novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

"Behind" means not running what the mesh would send

Browse Source 
It meant "failed or refused". So a machine that applied cleanly and whose
declaration has since changed was not behind — and novox/hq ADR 0010's
question, did my change go out?, was answerable exactly for the machines that
broke. For every machine that worked, the answer was silence whether the change
had gone out or not, which is the thing replacing a pipeline was supposed not
to cost.

The mesh now records a digest of what it last sent each machine. A digest
rather than the declaration: it can compute what a machine should be at any
moment, and keeping a copy would be a second account of it able to disagree
with the first. What cannot be recomputed is what was actually sent.

Recorded after the send, not before — a digest kept for something that failed
to send would make the machine look current for a declaration it never
received.

Never told stays separate from out of date. The remedy is the same push and the
situations are not alike: nobody has ever asked that machine to be anything.
And a machine the mesh could not work out is not reported as waiting, because
saying so would invent a comparison — that is `plan`'s answer to give.

`status` says it and `push --behind` sends it, or the flag would know something
the person reading the status does not.
This commit is contained in:
jschoubben
2026-08-31 05:17:21 +02:00
parent dfca21fa55
commit 5a28434ba8
4 changed files with 290 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-control/main.go
+122 -5
View File
@@ -9,7 +9,9 @@ package main
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"flag"
@@ -1626,6 +1628,24 @@ func pushCommand(ctx context.Context, args []string) error {
for _, d := range wrong {
needsOne[d.Node] = d
}
// **And every machine not running what the mesh would send it.** "Behind" used to mean
// only "failed or refused", so a machine that applied cleanly and whose declaration has
// since changed was not behind — and novox/hq ADR 0010's question, *did my change go
// out?*, was answerable only for the machines that broke.
would, err := wouldSend(ctx, inv, nodes)
if err != nil {
return err
}
waiting, err := inv.Waiting(ctx, would)
if err != nil {
return err
}
for _, m := range waiting {
if _, already := needsOne[m.Node]; already {
continue
}
needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"}
}
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
@@ -1666,7 +1686,11 @@ func pushCommand(ctx context.Context, args []string) error {
// A machine that has been failing the same way for a long time is not going to stop
// because it was asked again. Said, and pushed to anyway — refusing would leave no
// way to retry after fixing the cause, and this is a command somebody ran.
if since := time.Since(doing.At); since > 6*time.Hour {
//
// Only for machines that reported something. One that is merely waiting has no report
// to be old, and saying it had been failing since the zero time would be a sentence
// about nothing.
if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour {
fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+
"unlikely to be timing\n",
n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04"))
@@ -1705,6 +1729,15 @@ func pushCommand(ctx context.Context, args []string) error {
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
// After it is away, not before. A digest recorded for something that failed to send would
// make the machine look current for a declaration it never received.
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
return err
}
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
}
fmt.Printf("\n%d node(s) told\n", len(sending))
@@ -1766,6 +1799,13 @@ func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
return err
}
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources))
}
return nil
@@ -1867,11 +1907,36 @@ func statusCommand(ctx context.Context, args []string) error {
fmt.Println()
}
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 {
if len(asked.waiting) > 0 {
// The other half of "is anything out of date": a module behind its source says the
// catalogue is old, and this says a machine is — and only this one has somebody's change
// waiting inside it.
var told, never []string
for _, m := range asked.waiting {
if m.Never {
never = append(never, m.Node)
continue
}
told = append(told, m.Node)
}
if len(told) > 0 {
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
len(told), strings.Join(told, ", "))
}
if len(never) > 0 {
// Never told is not out of date. The remedy is the same push and the situation is
// not the same at all: nobody has ever asked this machine to be anything.
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
len(never), strings.Join(never, ", "))
}
fmt.Printf("\n `push --behind` sends them\n\n")
}
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means all three questions were asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, "+
"and every module current with its source\n", len(nodes))
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
"the mesh would send them, and every module current with its source\n", len(nodes))
}
return nil
}
@@ -2521,6 +2586,8 @@ type answers struct {
quiet []inventory.Node
behind map[string][]string
sources map[string]inventory.Source
// waiting is every machine not running what the mesh would send it.
waiting []inventory.Machine
}
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
@@ -2556,6 +2623,17 @@ func theThreeQuestions(ctx context.Context, inv *inventory.Inventory) (answers,
if err != nil {
return answers{}, err
}
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
// this one says a machine is — and only the second has anybody's change waiting in it.
would, err := wouldSend(ctx, inv, out.nodes)
if err != nil {
return answers{}, err
}
out.waiting, err = inv.Waiting(ctx, would)
if err != nil {
return answers{}, err
}
out.sources = map[string]inventory.Source{}
for module := range out.behind {
from, err := inv.SourceOf(ctx, module)
@@ -2619,3 +2697,42 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
}
return nil
}
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
//
// Over the same bytes that are sent, so the comparison is of the thing itself rather than of
// something derived beside it that could drift from it.
func digestOf(body []byte) string {
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:])
}
// wouldSend is the digest of what each machine should be right now.
//
// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
func wouldSend(ctx context.Context, inv *inventory.Inventory,
nodes []inventory.Node) (map[string]string, error) {
gens, err := generators(ctx, inv)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, inv, n.Name)
if err != nil {
continue
}
resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens)
if err != nil {
continue
}
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources})
if err != nil {
return nil, err
}
out[n.Name] = digestOf(body)
}
return out, nil
}