From c0107b85729e3bd2dc9592049fab4f29e0e5b474 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 1 Sep 2026 23:02:26 +0200 Subject: [PATCH] Status says when each machine last reported, beside when it was sent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Not waiting" says the declaration is current, not that the machine finished applying it: the sent digest is recorded at send. So a test that pushed, saw waiting clear, and asked the machine what it was running found containers that did not exist yet — the certificate fix made compositions stable, and the settling that used to fail first had been hiding the gap behind it. The mesh already held the missing half: every machine's last report, with its time. It just was not in the JSON. `reported` now sets each machine's last word beside when the current declaration went to it, and "has it caught up" becomes a comparison of two timestamps the mesh recorded itself — a report newer than the send means the machine acted on what was sent; older means it is still working, which waiting alone cannot distinguish. --- cmd/mesh-control/board.go | 2 +- cmd/mesh-control/build.go | 4 ++++ cmd/mesh-control/readable.go | 21 ++++++++++++++++-- cmd/mesh-control/readable_test.go | 4 ++-- cmd/mesh-control/status.go | 7 +++++- internal/inventory/nodes.go | 37 +++++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-control/board.go b/cmd/mesh-control/board.go index a9e25a3..fc11d2a 100644 --- a/cmd/mesh-control/board.go +++ b/cmd/mesh-control/board.go @@ -95,7 +95,7 @@ func board() http.Handler { return } body, err := statusAsJSON(asked.wrong, asked.nodes, asked.quiet, asked.behind, asked.sources, - asked.waiting) + asked.waiting, asked.reported) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/cmd/mesh-control/build.go b/cmd/mesh-control/build.go index dfb725d..6c7d9b8 100644 --- a/cmd/mesh-control/build.go +++ b/cmd/mesh-control/build.go @@ -438,4 +438,8 @@ type answers struct { sources map[string]inventory.Source // waiting is every machine not running what the mesh would send it. waiting []inventory.Machine + // reported is every machine's last word beside when it was last sent a declaration — the + // pair that answers "has it caught up", which waiting alone cannot (the sent digest is + // recorded at send, not at apply). + reported []inventory.Reported } diff --git a/cmd/mesh-control/readable.go b/cmd/mesh-control/readable.go index 9b17a0d..fabffaa 100644 --- a/cmd/mesh-control/readable.go +++ b/cmd/mesh-control/readable.go @@ -35,6 +35,11 @@ type meshStatus struct { // Behind one level down: that says the catalogue is old, this says a machine is — and only // this one has somebody's change waiting inside it. Waiting []machineWaiting `json:"waiting"` + // Reported is every machine's last word beside when it was last sent a declaration. A + // machine whose report is newer than its send has acted on the current declaration; one + // whose is older is still working — and Waiting cannot tell those apart, because the sent + // digest is recorded at send, not at apply. + Reported []machineReported `json:"reported"` // Machines is how many the mesh knows about, so a reader can tell "none wrong" from // "none at all". Machines int `json:"machines"` @@ -54,6 +59,13 @@ type machineDoing struct { At time.Time `json:"at"` } +type machineReported struct { + Node string `json:"node"` + Outcome string `json:"outcome"` + At *time.Time `json:"at,omitempty"` + Sent *time.Time `json:"sent,omitempty"` +} + type machineWaiting struct { Node string `json:"node"` // Never is true when nothing has ever been sent to it. Not out of date: nobody has ever asked @@ -79,10 +91,15 @@ type moduleBehind struct { // statusAsJSON answers the same three questions as the text form, from the same calls. func statusAsJSON(wrong []inventory.Doing, nodes []inventory.Node, quiet []inventory.Node, behind map[string][]string, sources map[string]inventory.Source, - waiting []inventory.Machine) ([]byte, error) { + waiting []inventory.Machine, reported []inventory.Reported) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, - Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}} + Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, + Reported: []machineReported{}} + for _, r := range reported { + out.Reported = append(out.Reported, machineReported{ + Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent}) + } for _, m := range waiting { out.Waiting = append(out.Waiting, machineWaiting{ diff --git a/cmd/mesh-control/readable_test.go b/cmd/mesh-control/readable_test.go index f4705d0..3d1504f 100644 --- a/cmd/mesh-control/readable_test.go +++ b/cmd/mesh-control/readable_test.go @@ -17,7 +17,7 @@ import ( func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node, behind map[string][]string, sources map[string]inventory.Source) map[string]any { t.Helper() - body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil) + body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil, nil) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) { body, err := statusAsJSON( []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused, Refused: "resource \"x\": a file needs a path"}}, - []inventory.Node{{Name: "a"}}, nil, nil, nil, nil) + []inventory.Node{{Name: "a"}}, nil, nil, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-control/status.go b/cmd/mesh-control/status.go index 5cd20ea..6c3bf49 100644 --- a/cmd/mesh-control/status.go +++ b/cmd/mesh-control/status.go @@ -54,7 +54,8 @@ func statusCommand(ctx context.Context, args []string) error { behind, sources := asked.behind, asked.sources if *asJSON { - body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting) + body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting, + asked.reported) if err != nil { return err } @@ -207,6 +208,10 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if err != nil { return answers{}, err } + out.reported, err = inv.LastReports(ctx) + if err != nil { + return answers{}, err + } out.sources = map[string]inventory.Source{} for module := range out.behind { from, err := inv.SourceOf(ctx, module) diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index b8d359f..baee3a3 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -514,6 +514,43 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro return d, true, nil } +// Reported is one machine's last word set beside what was last asked of it. +// +// The pair is what makes "has it caught up" answerable: a machine that reported *after* it was +// sent the current declaration has acted on it; one that reported before is still working, or +// has not started — and "not waiting" alone cannot tell those apart, because the sent digest is +// recorded at send, not at apply. The lab asserted on a machine the moment its declaration was +// current and found containers that did not exist yet. +type Reported struct { + Node string + Outcome string + // At is when it last reported; nil if it never has. + At *time.Time + // Sent is when the current declaration went to it; nil if nothing ever did. + Sent *time.Time +} + +// LastReports is every machine's last report beside when it was last sent a declaration. +func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { + rows, err := i.store.Pool().Query(ctx, + `select n.name, coalesce(r.outcome, ''), r.at, n.sent_at + from node n left join node_report r on r.node = n.id + order by n.name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Reported + for rows.Next() { + var r Reported + if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + // RecordSent keeps a digest of the declaration a machine was last sent. // // **A digest rather than the declaration.** The mesh can compute what a machine should be at any