From 5a28434ba8e18d5b0606dc7d64083d94a1cc2a4a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 31 Aug 2026 05:17:21 +0200 Subject: [PATCH] "Behind" means not running what the mesh would send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It meant "failed or refused". So a machine that applied cleanly and whose declaration has since changed was not behind — and novox/hq ADR 0010's question, did my change go out?, was answerable exactly for the machines that broke. For every machine that worked, the answer was silence whether the change had gone out or not, which is the thing replacing a pipeline was supposed not to cost. The mesh now records a digest of what it last sent each machine. A digest rather than the declaration: it can compute what a machine should be at any moment, and keeping a copy would be a second account of it able to disagree with the first. What cannot be recomputed is what was actually sent. Recorded after the send, not before — a digest kept for something that failed to send would make the machine look current for a declaration it never received. Never told stays separate from out of date. The remedy is the same push and the situations are not alike: nobody has ever asked that machine to be anything. And a machine the mesh could not work out is not reported as waiting, because saying so would invent a comparison — that is `plan`'s answer to give. `status` says it and `push --behind` sends it, or the flag would know something the person reading the status does not. --- cmd/mesh-control/main.go | 127 +++++++++++++++++- internal/inventory/doing_test.go | 90 +++++++++++++ .../0014-what-a-machine-was-last-sent.sql | 15 +++ internal/inventory/nodes.go | 63 +++++++++ 4 files changed, 290 insertions(+), 5 deletions(-) create mode 100644 internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 6d5f7bf..dd06c67 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -9,7 +9,9 @@ package main import ( "context" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "flag" @@ -1626,6 +1628,24 @@ func pushCommand(ctx context.Context, args []string) error { for _, d := range wrong { needsOne[d.Node] = d } + // **And every machine not running what the mesh would send it.** "Behind" used to mean + // only "failed or refused", so a machine that applied cleanly and whose declaration has + // since changed was not behind — and novox/hq ADR 0010's question, *did my change go + // out?*, was answerable only for the machines that broke. + would, err := wouldSend(ctx, inv, nodes) + if err != nil { + return err + } + waiting, err := inv.Waiting(ctx, would) + if err != nil { + return err + } + for _, m := range waiting { + if _, already := needsOne[m.Node]; already { + continue + } + needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} + } if len(needsOne) == 0 { // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" // must never look the same. @@ -1666,7 +1686,11 @@ func pushCommand(ctx context.Context, args []string) error { // A machine that has been failing the same way for a long time is not going to stop // because it was asked again. Said, and pushed to anyway — refusing would leave no // way to retry after fixing the cause, and this is a command somebody ran. - if since := time.Since(doing.At); since > 6*time.Hour { + // + // Only for machines that reported something. One that is merely waiting has no report + // to be old, and saying it had been failing since the zero time would be a sentence + // about nothing. + if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ "unlikely to be timing\n", n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) @@ -1705,6 +1729,15 @@ func pushCommand(ctx context.Context, args []string) error { if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { return err } + // After it is away, not before. A digest recorded for something that failed to send would + // make the machine look current for a declaration it never received. + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) } fmt.Printf("\n%d node(s) told\n", len(sending)) @@ -1766,6 +1799,13 @@ func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { return err } + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) } return nil @@ -1867,11 +1907,36 @@ func statusCommand(ctx context.Context, args []string) error { fmt.Println() } - if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 { + if len(asked.waiting) > 0 { + // The other half of "is anything out of date": a module behind its source says the + // catalogue is old, and this says a machine is — and only this one has somebody's change + // waiting inside it. + var told, never []string + for _, m := range asked.waiting { + if m.Never { + never = append(never, m.Node) + continue + } + told = append(told, m.Node) + } + if len(told) > 0 { + fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n", + len(told), strings.Join(told, ", ")) + } + if len(never) > 0 { + // Never told is not out of date. The remedy is the same push and the situation is + // not the same at all: nobody has ever asked this machine to be anything. + fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", + len(never), strings.Join(never, ", ")) + } + fmt.Printf("\n `push --behind` sends them\n\n") + } + + if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 { // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, - // and getting here means all three questions were asked and answered. - fmt.Printf("%d machine(s), all doing what they were told, all heard from, "+ - "and every module current with its source\n", len(nodes)) + // and getting here means every question was asked and answered. + fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+ + "the mesh would send them, and every module current with its source\n", len(nodes)) } return nil } @@ -2521,6 +2586,8 @@ type answers struct { quiet []inventory.Node behind map[string][]string sources map[string]inventory.Source + // waiting is every machine not running what the mesh would send it. + waiting []inventory.Machine } // theThreeQuestions reads what anything answering "is the mesh alright" needs. @@ -2556,6 +2623,17 @@ func theThreeQuestions(ctx context.Context, inv *inventory.Inventory) (answers, if err != nil { return answers{}, err } + // And which machines are not running what the mesh would send them. The same question as a + // module being behind its source, one level down: that one says the catalogue is out of date, + // this one says a machine is — and only the second has anybody's change waiting in it. + would, err := wouldSend(ctx, inv, out.nodes) + if err != nil { + return answers{}, err + } + out.waiting, err = inv.Waiting(ctx, would) + if err != nil { + return answers{}, err + } out.sources = map[string]inventory.Source{} for module := range out.behind { from, err := inv.SourceOf(ctx, module) @@ -2619,3 +2697,42 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error } return nil } + +// digestOf is what the mesh compares to answer "has this machine been sent what it should be". +// +// Over the same bytes that are sent, so the comparison is of the thing itself rather than of +// something derived beside it that could drift from it. +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +// wouldSend is the digest of what each machine should be right now. +// +// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot +// be worked out" is a different problem with a different remedy, and `plan` is where it is said. +func wouldSend(ctx context.Context, inv *inventory.Inventory, + nodes []inventory.Node) (map[string]string, error) { + + gens, err := generators(ctx, inv) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, n := range nodes { + plan, settings, err := planFor(ctx, inv, n.Name) + if err != nil { + continue + } + resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) + if err != nil { + continue + } + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources}) + if err != nil { + return nil, err + } + out[n.Name] = digestOf(body) + } + return out, nil +} diff --git a/internal/inventory/doing_test.go b/internal/inventory/doing_test.go index 44a9760..6ba3f5e 100644 --- a/internal/inventory/doing_test.go +++ b/internal/inventory/doing_test.go @@ -155,3 +155,93 @@ func TestWhatANodeSaidGoesWhenTheNodeDoes(t *testing.T) { t.Fatalf("%d report(s) outlived the machine", left) } } + +// "Behind" must mean not running what the mesh would send, not only "failed". +// +// novox/hq ADR 0010 names losing "did my change go out?" as the real risk of replacing a pipeline +// with a comparison. With behind meaning only failed-or-refused, that question was answerable +// exactly for the machines that broke — and for every machine that worked, the answer was silence +// whether the change had gone out or not. +func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + anchor, err := inv.AddNode(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(ctx, "laptop"); err != nil { + t.Fatal(err) + } + + // Never sent anything: waiting, and said differently. Nobody has ever asked it to be + // anything, which is not the same as it being out of date. + waiting, err := inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 2 { + t.Fatalf("machines that were never sent anything are not waiting: %+v", waiting) + } + for _, m := range waiting { + if !m.Never { + t.Fatalf("%s was never sent anything and does not say so: %+v", m.Node, m) + } + } + + // Sent what it should be: not waiting. + if err := inv.RecordSent(ctx, anchor.ID, "aaa"); err != nil { + t.Fatal(err) + } + waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 1 || waiting[0].Node != "laptop" { + t.Fatalf("a machine sent exactly what it should be is still waiting: %+v", waiting) + } + + // The declaration changes: waiting again, and no longer "never". + waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "ccc", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + var found bool + for _, m := range waiting { + if m.Node != "anchor" { + continue + } + found = true + if m.Never { + t.Fatal("a machine that has been sent something is reported as never told") + } + if m.Sent != "aaa" { + t.Fatalf("what it was last sent was lost: %+v", m) + } + } + if !found { + t.Fatal("a machine whose declaration changed since it was sent is not waiting") + } +} + +// A machine nobody worked out is not reported as waiting: saying so would invent a comparison. +func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + node, err := inv.AddNode(ctx, "unresolvable") + if err != nil { + t.Fatal(err) + } + // It has been sent something before, which is what makes this the case the guard is for: a + // machine with a digest and nothing computed for it would compare against the empty string + // and look out of date, when the truth is that nobody worked out what it should be. + if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time"); err != nil { + t.Fatal(err) + } + waiting, err := inv.Waiting(ctx, map[string]string{}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 0 { + t.Fatalf("a machine the caller could not work out was reported as waiting: %+v", waiting) + } +} diff --git a/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql b/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql new file mode 100644 index 0000000..039e27b --- /dev/null +++ b/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql @@ -0,0 +1,15 @@ +-- What the mesh last sent a machine, as a digest. +-- +-- Not the declaration itself: the mesh can compute that again at any moment, and keeping a copy +-- would be a second account of what a machine should be, able to disagree with the first. What +-- cannot be recomputed is *what was actually sent*, and that is the whole of the difference +-- between "this machine is out of date" and "this machine has never been told". +-- +-- Without it, "behind" could only mean "failed or refused" — so a machine that applied cleanly and +-- whose declaration has since changed was not behind, and novox/hq ADR 0010's question, *did my +-- change go out?*, was answerable only for machines that broke. + +alter table node add column sent text; +-- When, so a machine sent something long ago and silent since is distinguishable from one sent +-- something a moment ago that has not had time to answer. +alter table node add column sent_at timestamptz; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index d0811b1..b8d359f 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -513,3 +513,66 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro } return d, true, nil } + +// RecordSent keeps a digest of the declaration a machine was last sent. +// +// **A digest rather than the declaration.** The mesh can compute what a machine should be at any +// moment; keeping a copy would be a second account of it, able to disagree with the first. What +// cannot be recomputed is what was *actually sent*, and that is the whole difference between a +// machine that is out of date and one that has never been told. +func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error { + _, err := i.store.Pool().Exec(ctx, + `update node set sent = $2, sent_at = now() where id = $1`, node, digest) + return err +} + +// Waiting is every machine whose declaration has changed since it was last sent one. +// +// The caller works out what each machine should be now, because only it can — resolution is the +// control plane's and this context holds records. What is answered here is the comparison. +// +// **A machine that has never been sent anything is waiting**, and says so differently: it is not +// out of date, it has never been told, and the remedy is the same push while the situation is not +// the same at all. +func (i *Inventory) Waiting(ctx context.Context, would map[string]string) ([]Machine, error) { + rows, err := i.store.Pool().Query(ctx, + `select name, coalesce(sent, ''), sent_at from node order by name`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Machine + for rows.Next() { + var m Machine + var at *time.Time + if err := rows.Scan(&m.Node, &m.Sent, &at); err != nil { + return nil, err + } + m.SentAt = at + wanted, known := would[m.Node] + if !known { + // Nothing was computed for it — it resolves to nothing, or the caller did not ask. + // Silence rather than a guess: saying "waiting" about a machine nobody worked out + // would be inventing a comparison. + continue + } + if m.Sent == wanted { + continue + } + m.Never = m.Sent == "" + out = append(out, m) + } + return out, rows.Err() +} + +// Machine is one machine that has not been sent what it should be. +type Machine struct { + Node string + // Sent is the digest it last received, empty if it has never received one. + Sent string + SentAt *time.Time + // Never is true when it has never been sent anything, which is a different situation from + // being out of date and reads differently to whoever is looking. + Never bool +}