diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 6d5f7bf..dd06c67 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -9,7 +9,9 @@ package main import ( "context" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "flag" @@ -1626,6 +1628,24 @@ func pushCommand(ctx context.Context, args []string) error { for _, d := range wrong { needsOne[d.Node] = d } + // **And every machine not running what the mesh would send it.** "Behind" used to mean + // only "failed or refused", so a machine that applied cleanly and whose declaration has + // since changed was not behind — and novox/hq ADR 0010's question, *did my change go + // out?*, was answerable only for the machines that broke. + would, err := wouldSend(ctx, inv, nodes) + if err != nil { + return err + } + waiting, err := inv.Waiting(ctx, would) + if err != nil { + return err + } + for _, m := range waiting { + if _, already := needsOne[m.Node]; already { + continue + } + needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} + } if len(needsOne) == 0 { // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" // must never look the same. @@ -1666,7 +1686,11 @@ func pushCommand(ctx context.Context, args []string) error { // A machine that has been failing the same way for a long time is not going to stop // because it was asked again. Said, and pushed to anyway — refusing would leave no // way to retry after fixing the cause, and this is a command somebody ran. - if since := time.Since(doing.At); since > 6*time.Hour { + // + // Only for machines that reported something. One that is merely waiting has no report + // to be old, and saying it had been failing since the zero time would be a sentence + // about nothing. + if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ "unlikely to be timing\n", n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) @@ -1705,6 +1729,15 @@ func pushCommand(ctx context.Context, args []string) error { if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { return err } + // After it is away, not before. A digest recorded for something that failed to send would + // make the machine look current for a declaration it never received. + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) } fmt.Printf("\n%d node(s) told\n", len(sending)) @@ -1766,6 +1799,13 @@ func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { return err } + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) } return nil @@ -1867,11 +1907,36 @@ func statusCommand(ctx context.Context, args []string) error { fmt.Println() } - if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 { + if len(asked.waiting) > 0 { + // The other half of "is anything out of date": a module behind its source says the + // catalogue is old, and this says a machine is — and only this one has somebody's change + // waiting inside it. + var told, never []string + for _, m := range asked.waiting { + if m.Never { + never = append(never, m.Node) + continue + } + told = append(told, m.Node) + } + if len(told) > 0 { + fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n", + len(told), strings.Join(told, ", ")) + } + if len(never) > 0 { + // Never told is not out of date. The remedy is the same push and the situation is + // not the same at all: nobody has ever asked this machine to be anything. + fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", + len(never), strings.Join(never, ", ")) + } + fmt.Printf("\n `push --behind` sends them\n\n") + } + + if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 { // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, - // and getting here means all three questions were asked and answered. - fmt.Printf("%d machine(s), all doing what they were told, all heard from, "+ - "and every module current with its source\n", len(nodes)) + // and getting here means every question was asked and answered. + fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+ + "the mesh would send them, and every module current with its source\n", len(nodes)) } return nil } @@ -2521,6 +2586,8 @@ type answers struct { quiet []inventory.Node behind map[string][]string sources map[string]inventory.Source + // waiting is every machine not running what the mesh would send it. + waiting []inventory.Machine } // theThreeQuestions reads what anything answering "is the mesh alright" needs. @@ -2556,6 +2623,17 @@ func theThreeQuestions(ctx context.Context, inv *inventory.Inventory) (answers, if err != nil { return answers{}, err } + // And which machines are not running what the mesh would send them. The same question as a + // module being behind its source, one level down: that one says the catalogue is out of date, + // this one says a machine is — and only the second has anybody's change waiting in it. + would, err := wouldSend(ctx, inv, out.nodes) + if err != nil { + return answers{}, err + } + out.waiting, err = inv.Waiting(ctx, would) + if err != nil { + return answers{}, err + } out.sources = map[string]inventory.Source{} for module := range out.behind { from, err := inv.SourceOf(ctx, module) @@ -2619,3 +2697,42 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error } return nil } + +// digestOf is what the mesh compares to answer "has this machine been sent what it should be". +// +// Over the same bytes that are sent, so the comparison is of the thing itself rather than of +// something derived beside it that could drift from it. +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +// wouldSend is the digest of what each machine should be right now. +// +// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot +// be worked out" is a different problem with a different remedy, and `plan` is where it is said. +func wouldSend(ctx context.Context, inv *inventory.Inventory, + nodes []inventory.Node) (map[string]string, error) { + + gens, err := generators(ctx, inv) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, n := range nodes { + plan, settings, err := planFor(ctx, inv, n.Name) + if err != nil { + continue + } + resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) + if err != nil { + continue + } + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources}) + if err != nil { + return nil, err + } + out[n.Name] = digestOf(body) + } + return out, nil +} diff --git a/internal/inventory/doing_test.go b/internal/inventory/doing_test.go index 44a9760..6ba3f5e 100644 --- a/internal/inventory/doing_test.go +++ b/internal/inventory/doing_test.go @@ -155,3 +155,93 @@ func TestWhatANodeSaidGoesWhenTheNodeDoes(t *testing.T) { t.Fatalf("%d report(s) outlived the machine", left) } } + +// "Behind" must mean not running what the mesh would send, not only "failed". +// +// novox/hq ADR 0010 names losing "did my change go out?" as the real risk of replacing a pipeline +// with a comparison. With behind meaning only failed-or-refused, that question was answerable +// exactly for the machines that broke — and for every machine that worked, the answer was silence +// whether the change had gone out or not. +func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + anchor, err := inv.AddNode(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(ctx, "laptop"); err != nil { + t.Fatal(err) + } + + // Never sent anything: waiting, and said differently. Nobody has ever asked it to be + // anything, which is not the same as it being out of date. + waiting, err := inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 2 { + t.Fatalf("machines that were never sent anything are not waiting: %+v", waiting) + } + for _, m := range waiting { + if !m.Never { + t.Fatalf("%s was never sent anything and does not say so: %+v", m.Node, m) + } + } + + // Sent what it should be: not waiting. + if err := inv.RecordSent(ctx, anchor.ID, "aaa"); err != nil { + t.Fatal(err) + } + waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 1 || waiting[0].Node != "laptop" { + t.Fatalf("a machine sent exactly what it should be is still waiting: %+v", waiting) + } + + // The declaration changes: waiting again, and no longer "never". + waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "ccc", "laptop": "bbb"}) + if err != nil { + t.Fatal(err) + } + var found bool + for _, m := range waiting { + if m.Node != "anchor" { + continue + } + found = true + if m.Never { + t.Fatal("a machine that has been sent something is reported as never told") + } + if m.Sent != "aaa" { + t.Fatalf("what it was last sent was lost: %+v", m) + } + } + if !found { + t.Fatal("a machine whose declaration changed since it was sent is not waiting") + } +} + +// A machine nobody worked out is not reported as waiting: saying so would invent a comparison. +func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + node, err := inv.AddNode(ctx, "unresolvable") + if err != nil { + t.Fatal(err) + } + // It has been sent something before, which is what makes this the case the guard is for: a + // machine with a digest and nothing computed for it would compare against the empty string + // and look out of date, when the truth is that nobody worked out what it should be. + if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time"); err != nil { + t.Fatal(err) + } + waiting, err := inv.Waiting(ctx, map[string]string{}) + if err != nil { + t.Fatal(err) + } + if len(waiting) != 0 { + t.Fatalf("a machine the caller could not work out was reported as waiting: %+v", waiting) + } +} diff --git a/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql b/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql new file mode 100644 index 0000000..039e27b --- /dev/null +++ b/internal/inventory/migrations/0014-what-a-machine-was-last-sent.sql @@ -0,0 +1,15 @@ +-- What the mesh last sent a machine, as a digest. +-- +-- Not the declaration itself: the mesh can compute that again at any moment, and keeping a copy +-- would be a second account of what a machine should be, able to disagree with the first. What +-- cannot be recomputed is *what was actually sent*, and that is the whole of the difference +-- between "this machine is out of date" and "this machine has never been told". +-- +-- Without it, "behind" could only mean "failed or refused" — so a machine that applied cleanly and +-- whose declaration has since changed was not behind, and novox/hq ADR 0010's question, *did my +-- change go out?*, was answerable only for machines that broke. + +alter table node add column sent text; +-- When, so a machine sent something long ago and silent since is distinguishable from one sent +-- something a moment ago that has not had time to answer. +alter table node add column sent_at timestamptz; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index d0811b1..b8d359f 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -513,3 +513,66 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro } return d, true, nil } + +// RecordSent keeps a digest of the declaration a machine was last sent. +// +// **A digest rather than the declaration.** The mesh can compute what a machine should be at any +// moment; keeping a copy would be a second account of it, able to disagree with the first. What +// cannot be recomputed is what was *actually sent*, and that is the whole difference between a +// machine that is out of date and one that has never been told. +func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error { + _, err := i.store.Pool().Exec(ctx, + `update node set sent = $2, sent_at = now() where id = $1`, node, digest) + return err +} + +// Waiting is every machine whose declaration has changed since it was last sent one. +// +// The caller works out what each machine should be now, because only it can — resolution is the +// control plane's and this context holds records. What is answered here is the comparison. +// +// **A machine that has never been sent anything is waiting**, and says so differently: it is not +// out of date, it has never been told, and the remedy is the same push while the situation is not +// the same at all. +func (i *Inventory) Waiting(ctx context.Context, would map[string]string) ([]Machine, error) { + rows, err := i.store.Pool().Query(ctx, + `select name, coalesce(sent, ''), sent_at from node order by name`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Machine + for rows.Next() { + var m Machine + var at *time.Time + if err := rows.Scan(&m.Node, &m.Sent, &at); err != nil { + return nil, err + } + m.SentAt = at + wanted, known := would[m.Node] + if !known { + // Nothing was computed for it — it resolves to nothing, or the caller did not ask. + // Silence rather than a guess: saying "waiting" about a machine nobody worked out + // would be inventing a comparison. + continue + } + if m.Sent == wanted { + continue + } + m.Never = m.Sent == "" + out = append(out, m) + } + return out, rows.Err() +} + +// Machine is one machine that has not been sent what it should be. +type Machine struct { + Node string + // Sent is the digest it last received, empty if it has never received one. + Sent string + SentAt *time.Time + // Never is true when it has never been sent anything, which is a different situation from + // being out of date and reads differently to whoever is looking. + Never bool +}