Merge pull request 'Say each machine's last send in status and node show (hq issue 485)' (#234) from fix/485-the-last-send-is-said into main
This commit was merged in pull request #234.
This commit is contained in:
@@ -866,6 +866,10 @@ type answers struct {
|
|||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||||
pending []inventory.PendingAssignment
|
pending []inventory.PendingAssignment
|
||||||
pendingUnread string
|
pendingUnread string
|
||||||
|
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||||
|
// issue 485); lastSentUnread why it could not be read.
|
||||||
|
lastSent map[string]inventory.Send
|
||||||
|
lastSentUnread string
|
||||||
}
|
}
|
||||||
|
|
||||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
|||||||
@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
|
|||||||
return strings.Join(modules, ", ")
|
return strings.Join(modules, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
|
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
||||||
// nothing when the mesh has not recorded a send to it.
|
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
||||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||||
s, found, err := inv.LastSend(ctx, node)
|
s, found, err := inv.LastSend(ctx, node)
|
||||||
if err != nil || !found {
|
if err != nil || !found {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
generation := "no generation recorded"
|
fmt.Fprintf(w, "%s:\n", node)
|
||||||
if s.Generation > 0 {
|
for _, line := range lastSendLines(s) {
|
||||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
fmt.Fprintln(w, line)
|
||||||
}
|
|
||||||
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
|
|
||||||
if s.RefusedAt != nil {
|
|
||||||
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
|
|
||||||
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
||||||
|
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
||||||
|
// machine answered of it — the refusal in its own words.
|
||||||
|
func lastSendLines(s inventory.Send) []string {
|
||||||
|
const stamp = "2006-01-02 15:04:05"
|
||||||
|
epoch := "under no lease epoch"
|
||||||
|
if s.Epoch > 0 {
|
||||||
|
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
||||||
|
}
|
||||||
|
generation := "no assignment generation recorded"
|
||||||
|
if s.Generation > 0 {
|
||||||
|
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||||
|
}
|
||||||
|
lines := []string{
|
||||||
|
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
||||||
|
fmt.Sprintf(" by %s", s.Sender),
|
||||||
|
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
||||||
|
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
||||||
|
}
|
||||||
|
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
||||||
|
switch a := s.Answer; {
|
||||||
|
case s.RefusedAt != nil:
|
||||||
|
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
||||||
|
// answer to this send in particular.
|
||||||
|
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
||||||
|
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
||||||
|
case a.Outcome == "" || a.At == nil:
|
||||||
|
answer("the machine has not reported on it yet")
|
||||||
|
case a.Outcome == inventory.OutcomeApplied:
|
||||||
|
answer("applied it at " + a.At.Local().Format(stamp))
|
||||||
|
case a.Outcome == inventory.OutcomeRefused:
|
||||||
|
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
||||||
|
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
||||||
|
lines = append(lines, " "+strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
||||||
|
a.At.Local().Format(stamp), a.Failed))
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
||||||
|
func sendAnswerWord(s inventory.Send) string {
|
||||||
|
switch a := s.Answer; {
|
||||||
|
case s.RefusedAt != nil:
|
||||||
|
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
||||||
|
case a.Outcome == "" || a.At == nil:
|
||||||
|
return "not reported on yet"
|
||||||
|
case a.Outcome == inventory.OutcomeFailed:
|
||||||
|
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
||||||
|
default:
|
||||||
|
return a.Outcome
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
||||||
|
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
||||||
|
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
||||||
|
if unread != "" {
|
||||||
|
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Println("last sent, per machine:")
|
||||||
|
for _, n := range nodes {
|
||||||
|
s, found := sends[n.Name]
|
||||||
|
if !found {
|
||||||
|
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
generation := "no generation"
|
||||||
|
if s.Generation > 0 {
|
||||||
|
generation = fmt.Sprintf("generation %d", s.Generation)
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
||||||
|
generation, sendAnswerWord(s))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
||||||
|
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
||||||
|
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
||||||
|
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
||||||
|
|
||||||
|
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
||||||
|
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
||||||
|
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
||||||
|
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||||
|
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||||
|
asked := answers{
|
||||||
|
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
||||||
|
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
||||||
|
Outcome: inventory.OutcomeApplied, At: &at})},
|
||||||
|
}
|
||||||
|
shown := printed(t, func() error { return printStatus(asked) })
|
||||||
|
var laptop, anchor []string
|
||||||
|
for _, line := range strings.Split(shown, "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) > 0 && fields[0] == "laptop" {
|
||||||
|
laptop = append(laptop, line)
|
||||||
|
}
|
||||||
|
if len(fields) > 0 && fields[0] == "anchor" {
|
||||||
|
anchor = append(anchor, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
||||||
|
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
||||||
|
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
||||||
|
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
||||||
|
}
|
||||||
|
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
||||||
|
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
||||||
|
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
||||||
|
lastSentUnread: "the store went away"}
|
||||||
|
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
||||||
|
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Error("a mesh whose sends could not be read is called well")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||||
|
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||||
|
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
||||||
|
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
||||||
|
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
||||||
|
if !strings.Contains(refused, want) {
|
||||||
|
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
||||||
|
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
||||||
|
t.Errorf("an applied send is not said applied:\n%s", applied)
|
||||||
|
}
|
||||||
|
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
||||||
|
"not reported on it yet") {
|
||||||
|
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
||||||
|
}
|
||||||
|
stale := aLastSend(inventory.SendAnswer{})
|
||||||
|
when := at
|
||||||
|
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
||||||
|
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
||||||
|
!strings.Contains(s, "generation 44") {
|
||||||
|
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
||||||
|
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||||
|
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||||
|
Refused: "unknown field", At: &at})}}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
LastSent []map[string]any `json:"lastSent"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.LastSent) != 1 {
|
||||||
|
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
||||||
|
}
|
||||||
|
s := got.LastSent[0]
|
||||||
|
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
||||||
|
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
||||||
|
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
||||||
|
s["answeredAt"] == nil {
|
||||||
|
t.Fatalf("laptop's last send reads %v", s)
|
||||||
|
}
|
||||||
|
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
||||||
|
t.Fatalf("the modules it named read %v", s["modules"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
||||||
|
// read back.
|
||||||
|
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
||||||
|
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||||
|
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
||||||
|
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
||||||
|
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
||||||
|
"docker, sudo", "unknown field \"generation\""} {
|
||||||
|
if !strings.Contains(shown, want) {
|
||||||
|
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
||||||
|
var got struct {
|
||||||
|
LastSent []struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Sender string `json:"sender"`
|
||||||
|
Answer string `json:"answer"`
|
||||||
|
} `json:"lastSent"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||||
|
t.Fatalf("%v:\n%s", err, body)
|
||||||
|
}
|
||||||
|
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
||||||
|
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
||||||
|
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -804,6 +804,19 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
|||||||
if err := showMode(ctx, inv, node); err != nil {
|
if err := showMode(ctx, inv, node); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
||||||
|
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
||||||
|
// Unreadable is said, not taken for none.
|
||||||
|
switch last, found, err := inv.LastSend(ctx, name); {
|
||||||
|
case err != nil:
|
||||||
|
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
||||||
|
case !found:
|
||||||
|
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
||||||
|
default:
|
||||||
|
for _, line := range lastSendLines(last) {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||||
fmt.Println(line)
|
fmt.Println(line)
|
||||||
|
|||||||
@@ -109,6 +109,33 @@ type meshStatus struct {
|
|||||||
// why they could not be read.
|
// why they could not be read.
|
||||||
Pending []pendingStatus `json:"pending,omitempty"`
|
Pending []pendingStatus `json:"pending,omitempty"`
|
||||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||||
|
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
||||||
|
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
||||||
|
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
||||||
|
LastSent []machineLastSent `json:"lastSent"`
|
||||||
|
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineLastSent is one machine's last send as status says it.
|
||||||
|
type machineLastSent struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Sequence int64 `json:"sequence"`
|
||||||
|
SentAt time.Time `json:"sentAt"`
|
||||||
|
Sender string `json:"sender"`
|
||||||
|
Epoch int64 `json:"epoch"`
|
||||||
|
Generation int64 `json:"generation"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
Modules []string `json:"modules"`
|
||||||
|
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
||||||
|
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
||||||
|
Answer string `json:"answer"`
|
||||||
|
Refused string `json:"refused,omitempty"`
|
||||||
|
Failed int `json:"failed,omitempty"`
|
||||||
|
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
||||||
|
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
||||||
|
// it applied, and AppliedGeneration that generation; absent when it did not.
|
||||||
|
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
||||||
|
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingStatus is one pending assignment as status says it.
|
// pendingStatus is one pending assignment as status says it.
|
||||||
@@ -267,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||||
}
|
}
|
||||||
|
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
@@ -323,3 +351,20 @@ func say(value any) error {
|
|||||||
fmt.Println(string(body))
|
fmt.Println(string(body))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
||||||
|
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
||||||
|
out := []machineLastSent{}
|
||||||
|
for _, s := range sends {
|
||||||
|
modules := s.Modules
|
||||||
|
if modules == nil {
|
||||||
|
modules = []string{}
|
||||||
|
}
|
||||||
|
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
||||||
|
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
||||||
|
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
||||||
|
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -341,6 +341,10 @@ func printStatus(asked answers) error {
|
|||||||
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
|
||||||
|
// break "all well"; a record that could not be read does.
|
||||||
|
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
|
||||||
|
|
||||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
@@ -485,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
out.pendingUnread = err.Error()
|
out.pendingUnread = err.Error()
|
||||||
err = nil
|
err = nil
|
||||||
}
|
}
|
||||||
|
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
|
||||||
|
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
|
||||||
|
if out.lastSent, err = inv.LastSends(ctx); err != nil {
|
||||||
|
out.lastSentUnread = err.Error()
|
||||||
|
err = nil
|
||||||
|
}
|
||||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
@@ -617,7 +627,8 @@ func (a answers) well() bool {
|
|||||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
||||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
|
||||||
|
a.lastSentUnread == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
||||||
|
|||||||
@@ -100,6 +100,23 @@ type Send struct {
|
|||||||
CounterRaisedTo int64
|
CounterRaisedTo int64
|
||||||
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
|
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
|
||||||
Recorded bool
|
Recorded bool
|
||||||
|
// Answer is what the machine last reported of this send; filled only where a last send is read back
|
||||||
|
// (LastSend, LastSends), the zero answer elsewhere.
|
||||||
|
Answer SendAnswer
|
||||||
|
}
|
||||||
|
|
||||||
|
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
|
||||||
|
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
|
||||||
|
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
|
||||||
|
type SendAnswer struct {
|
||||||
|
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
|
||||||
|
Outcome string
|
||||||
|
// Refused is the machine's own words when it refused the declaration whole.
|
||||||
|
Refused string
|
||||||
|
// Failed is how many of its resources failed, when some did.
|
||||||
|
Failed int
|
||||||
|
// At is when the machine reported it; nil when it has not.
|
||||||
|
At *time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
|
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
|
||||||
@@ -209,20 +226,56 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
|
|||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal
|
// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
|
||||||
// of a sequence the mesh has no send for is not a send, and is not it.
|
// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
|
||||||
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
|
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
|
||||||
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
|
sends, err := i.lastSends(ctx, `n.name = $1`, name)
|
||||||
from declaration_send s join node n on n.id = s.node
|
|
||||||
where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Send{}, false, err
|
return Send{}, false, err
|
||||||
}
|
}
|
||||||
sends, err := scanSends(rows)
|
s, found := sends[name]
|
||||||
if err != nil || len(sends) == 0 {
|
return s, found, nil
|
||||||
return Send{}, false, err
|
}
|
||||||
|
|
||||||
|
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
|
||||||
|
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
|
||||||
|
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
|
||||||
|
return i.lastSends(ctx, `true`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
|
||||||
|
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
|
||||||
|
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
|
||||||
|
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
|
||||||
|
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
|
||||||
|
// whose report lands before the send's row is stamped.
|
||||||
|
// - from an engine that reports no order, the report came after the send — both times the store's own — so
|
||||||
|
// a machine sent again a declaration it once applied does not read as having applied the new send.
|
||||||
|
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
|
||||||
|
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
|
||||||
|
from declaration_send s join node n on n.id = s.node
|
||||||
|
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
|
||||||
|
and case when r.reported_sequence is not null
|
||||||
|
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
|
||||||
|
else r.at >= s.sent_at end
|
||||||
|
where s.recorded and `+where+`
|
||||||
|
order by n.name, s.id desc`, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return sends[0], true, nil
|
defer rows.Close()
|
||||||
|
out := map[string]Send{}
|
||||||
|
for rows.Next() {
|
||||||
|
var s Send
|
||||||
|
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
|
||||||
|
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
|
||||||
|
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[s.NodeName] = s
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
|
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
|
||||||
|
|||||||
@@ -205,3 +205,144 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
|
||||||
|
// words, or not answered yet — read from the machine's report only when that report is about the send's
|
||||||
|
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
|
||||||
|
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
|
||||||
|
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
|
||||||
|
inv, node := aNodeWithModules(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
record, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
send := func(sequence int64, digest string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
||||||
|
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
|
||||||
|
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report := func(d Doing) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
answer := func() SendAnswer {
|
||||||
|
t.Helper()
|
||||||
|
last, found, err := inv.LastSend(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
||||||
|
}
|
||||||
|
return last.Answer
|
||||||
|
}
|
||||||
|
|
||||||
|
send(11, "d11")
|
||||||
|
if a := answer(); a.Outcome != "" || a.At != nil {
|
||||||
|
t.Fatalf("a send not reported on reads as answered: %+v", a)
|
||||||
|
}
|
||||||
|
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
|
||||||
|
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
|
||||||
|
t.Fatalf("a send the machine reported applying reads %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
send(12, "d12")
|
||||||
|
if a := answer(); a.Outcome != "" {
|
||||||
|
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
|
||||||
|
}
|
||||||
|
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
|
||||||
|
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
|
||||||
|
t.Fatalf("a send the machine refused reads %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sent again a declaration it reported applying before the send: not answered until it reports again.
|
||||||
|
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
|
||||||
|
send(13, "d12")
|
||||||
|
if a := answer(); a.Outcome != "" {
|
||||||
|
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
all, err := inv.LastSends(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
|
||||||
|
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
|
||||||
|
t.Fatalf("every machine's last send reads %+v", all)
|
||||||
|
}
|
||||||
|
if _, found := all["laptop"]; found {
|
||||||
|
t.Error("a machine never sent anything has a last send")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A report about another declaration, or another send of the same one, is not the last send's answer**
|
||||||
|
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
|
||||||
|
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
|
||||||
|
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
|
||||||
|
// the report's epoch and sequence say which send it is about.
|
||||||
|
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
|
||||||
|
inv, node := aNodeWithModules(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
record, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
send := func(sequence int64, digest string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
||||||
|
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ordered := func(epoch, sequence int64, digest string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
|
||||||
|
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
answer := func() string {
|
||||||
|
t.Helper()
|
||||||
|
last, found, err := inv.LastSend(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
||||||
|
}
|
||||||
|
return last.Answer.Outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
// An engine that reports no order: the digest decides.
|
||||||
|
send(11, "d11")
|
||||||
|
send(12, "d12")
|
||||||
|
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a := answer(); a != "" {
|
||||||
|
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
|
||||||
|
send(20, "d20")
|
||||||
|
ordered(57, 20, "d20")
|
||||||
|
if a := answer(); a != OutcomeApplied {
|
||||||
|
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
|
||||||
|
}
|
||||||
|
send(21, "d20")
|
||||||
|
ordered(57, 20, "d20")
|
||||||
|
if a := answer(); a != "" {
|
||||||
|
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
|
||||||
|
}
|
||||||
|
ordered(56, 21, "d20")
|
||||||
|
if a := answer(); a != "" {
|
||||||
|
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
|
||||||
|
}
|
||||||
|
ordered(57, 21, "d20")
|
||||||
|
if a := answer(); a != OutcomeApplied {
|
||||||
|
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Vendored
+3
@@ -1,3 +1,6 @@
|
|||||||
# The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs
|
# The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs
|
||||||
# each one named here, by package, and fails unless every one passed — a test that needs the store skips without
|
# each one named here, by package, and fails unless every one passed — a test that needs the store skips without
|
||||||
# one, and a skip passes unseen. One per line: <package> <TestName>. Lines starting with # are said nothing of.
|
# one, and a skip passes unseen. One per line: <package> <TestName>. Lines starting with # are said nothing of.
|
||||||
|
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
|
||||||
|
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
|
||||||
|
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore
|
||||||
|
|||||||
Reference in New Issue
Block a user