diff --git a/cmd/mesh-control/board.go b/cmd/mesh-control/board.go
index fc11d2a..49702ac 100644
--- a/cmd/mesh-control/board.go
+++ b/cmd/mesh-control/board.go
@@ -7,6 +7,8 @@ import (
"fmt"
"html/template"
"net/http"
+ "sort"
+ "strings"
"time"
)
@@ -94,8 +96,7 @@ func board() http.Handler {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
- body, err := statusAsJSON(asked.wrong, asked.nodes, asked.quiet, asked.behind, asked.sources,
- asked.waiting, asked.reported)
+ body, err := statusAsJSON(asked)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -129,7 +130,21 @@ type view struct {
Quiet []quietMachine
Behind []staleModule
Waiting []waitingMachine
- At string
+ // Unresolved is every machine that cannot be worked out at all. Shown above everything else,
+ // because a machine here is in none of the other lists: nothing was computed for it, so it is
+ // not broken, not quiet and not behind — and a page without this said "all well" about a mesh
+ // where nothing could be sent anywhere.
+ Unresolved []blockedMachine
+ // Network is why the private network could not be computed, when it could not.
+ Network string
+ At string
+}
+
+type blockedMachine struct {
+ Node string
+ // Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
+ // a machine is usually blocked by more than one and fixing one of them changes nothing.
+ Said []string
}
type waitingMachine struct {
@@ -165,7 +180,20 @@ type staleModule struct {
}
func viewOf(asked answers) view {
- out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05")}
+ out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
+ Network: asked.network}
+ var blocked []string
+ for name := range asked.refused {
+ blocked = append(blocked, name)
+ }
+ sort.Strings(blocked)
+ for _, name := range blocked {
+ one := blockedMachine{Node: name}
+ for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
+ one.Said = append(one.Said, strings.TrimSpace(line))
+ }
+ out.Unresolved = append(out.Unresolved, one)
+ }
for _, d := range asked.wrong {
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
When: d.At.Local().Format("2006-01-02 15:04")}
@@ -223,6 +251,22 @@ find out.
{{else}}
{{.Machines}} machine{{if ne .Machines 1}}s{{end}}
+{{if .Unresolved}}
+Can everything be worked out?
+
+ {{range .Unresolved}}
+ - blocked {{.Node}}
+ {{range .Said}}
{{.}}
{{end}}
+
+ {{end}}
+
+Nothing can be sent to a machine here, and it appears in none of the lists below:
+nothing was computed for it, so there is nothing it can be behind.
+{{end}}
+{{if .Network}}
+The private network could not be computed: {{.Network}}
+{{end}}
+
Is anything broken?
{{if .Broken}}
diff --git a/cmd/mesh-control/build.go b/cmd/mesh-control/build.go
index 6c7d9b8..4204f39 100644
--- a/cmd/mesh-control/build.go
+++ b/cmd/mesh-control/build.go
@@ -442,4 +442,13 @@ type answers struct {
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
// recorded at send, not at apply).
reported []inventory.Reported
+ // refused is why a machine cannot be worked out at all, by name. A different thing from every
+ // other answer here: those are about a machine that was told something, and this is about one
+ // that cannot be told anything — it never reaches waiting, because nothing was computed for it
+ // to compare against, so without this a wholly blocked mesh reads as a well one.
+ refused map[string]string
+ // network is why the private network could not be computed, when it could not. Almost always
+ // a consequence of the refusals above: a node that does not resolve is not on the network, and
+ // a mesh whose hub is that node has no hub.
+ network string
}
diff --git a/cmd/mesh-control/readable.go b/cmd/mesh-control/readable.go
index 346e93c..68c8bc6 100644
--- a/cmd/mesh-control/readable.go
+++ b/cmd/mesh-control/readable.go
@@ -3,9 +3,8 @@ package main
import (
"encoding/json"
"fmt"
+ "sort"
"time"
-
- "github.com/novox/mesh-control/internal/inventory"
)
// The same answers, in a shape something other than a person can read.
@@ -40,11 +39,30 @@ type meshStatus struct {
// whose is older is still working — and Waiting cannot tell those apart, because the sent
// digest is recorded at send, not at apply.
Reported []machineReported `json:"reported"`
+ // Unresolved is every machine that cannot be worked out at all, with what the mesh said when
+ // it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
+ // there is nothing to compare it against and nothing it can be behind — which is why a
+ // document without this field described a wholly blocked mesh as a well one.
+ //
+ // Per machine, and data. One node failing must never take the document away from a reader
+ // asking about the others.
+ Unresolved []machineUnresolved `json:"unresolved"`
+ // Network is why the private network could not be computed, when it could not; absent when it
+ // could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
+ // network, and a mesh whose hub is that node has no hub.
+ Network string `json:"network,omitempty"`
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
// "none at all".
Machines int `json:"machines"`
}
+type machineUnresolved struct {
+ Node string `json:"node"`
+ // Problem is the mesh's own words, whole — newlines and all. It lists every requirement that
+ // could not be met, and a first line alone would name one of them and hide the rest.
+ Problem string `json:"problem"`
+}
+
type machineDoing struct {
Node string `json:"node"`
// Outcome is refused or failed. Kept distinct all the way out: they are fixed in different
@@ -92,14 +110,32 @@ type moduleBehind struct {
On []string `json:"on"`
}
-// statusAsJSON answers the same three questions as the text form, from the same calls.
-func statusAsJSON(wrong []inventory.Doing, nodes []inventory.Node, quiet []inventory.Node,
- behind map[string][]string, sources map[string]inventory.Source,
- waiting []inventory.Machine, reported []inventory.Reported) ([]byte, error) {
+// statusAsJSON answers the same questions as the text form, from the same reading.
+//
+// **It takes the whole reading rather than a growing argument list**, which is what let a new
+// answer be added to the text form and forgotten here — the two are one function's output in two
+// shapes, and they must not be able to differ about what was asked.
+//
+// It never fails on account of the mesh. Every per-machine problem in here is a field, so one
+// machine that cannot be worked out cannot stop a caller reading about the others: a
+// machine-readable interface that stops being machine-readable exactly when something is wrong is
+// one nobody can build an alarm on.
+func statusAsJSON(asked answers) ([]byte, error) {
+ wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
+ behind, sources := asked.behind, asked.sources
+ waiting, reported := asked.waiting, asked.reported
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
- Reported: []machineReported{}}
+ Reported: []machineReported{}, Unresolved: []machineUnresolved{},
+ Network: asked.network}
+ for name := range asked.refused {
+ out.Unresolved = append(out.Unresolved, machineUnresolved{
+ Node: name, Problem: asked.refused[name]})
+ }
+ sort.Slice(out.Unresolved, func(i, j int) bool {
+ return out.Unresolved[i].Node < out.Unresolved[j].Node
+ })
for _, r := range reported {
out.Reported = append(out.Reported, machineReported{
Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
diff --git a/cmd/mesh-control/readable_test.go b/cmd/mesh-control/readable_test.go
index 3d1504f..2091d2a 100644
--- a/cmd/mesh-control/readable_test.go
+++ b/cmd/mesh-control/readable_test.go
@@ -17,7 +17,8 @@ import (
func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
behind map[string][]string, sources map[string]inventory.Source) map[string]any {
t.Helper()
- body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil, nil)
+ body, err := statusAsJSON(answers{
+ wrong: wrong, nodes: nodes, quiet: quiet, behind: behind, sources: sources})
if err != nil {
t.Fatal(err)
}
@@ -123,10 +124,12 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) {
// Everything here comes from the mesh's own records, which hold no readable secret — but a
// shape a page is built against is exactly where one would eventually be added for
// convenience, so this says it out loud.
- body, err := statusAsJSON(
- []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
+ body, err := statusAsJSON(answers{
+ wrong: []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
Refused: "resource \"x\": a file needs a path"}},
- []inventory.Node{{Name: "a"}}, nil, nil, nil, nil, nil)
+ nodes: []inventory.Node{{Name: "a"}},
+ refused: map[string]string{"a": "nothing provides \"database\", wanted by web"},
+ })
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/mesh-control/status.go b/cmd/mesh-control/status.go
index 6c3bf49..86bd83a 100644
--- a/cmd/mesh-control/status.go
+++ b/cmd/mesh-control/status.go
@@ -9,6 +9,7 @@ import (
"time"
"github.com/novox/mesh-control/internal/inventory"
+ "github.com/novox/mesh-control/internal/overlay"
)
// is anything broken, is anything not answering, is anything out of date.
@@ -54,8 +55,7 @@ func statusCommand(ctx context.Context, args []string) error {
behind, sources := asked.behind, asked.sources
if *asJSON {
- body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting,
- asked.reported)
+ body, err := statusAsJSON(asked)
if err != nil {
return err
}
@@ -63,6 +63,30 @@ func statusCommand(ctx context.Context, args []string) error {
return nil
}
+ if len(asked.refused) > 0 {
+ // First, above everything else. A machine that cannot be worked out is not running an old
+ // declaration — it has no declaration, and nothing below this line is about it.
+ var names []string
+ for name := range asked.refused {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
+ len(names))
+ for _, name := range names {
+ fmt.Printf(" %s\n", name)
+ for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
+ fmt.Printf(" %s\n", strings.TrimSpace(line))
+ }
+ }
+ fmt.Println()
+ }
+
+ if asked.network != "" {
+ fmt.Printf("the private network could not be computed:\n %s\n\n",
+ strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
+ }
+
if len(wrong) > 0 {
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
for _, d := range wrong {
@@ -139,7 +163,8 @@ func statusCommand(ctx context.Context, args []string) error {
fmt.Printf("\n `push --behind` sends them\n\n")
}
- if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 {
+ if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
+ len(asked.refused) == 0 && asked.network == "" {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
@@ -197,12 +222,31 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
if err != nil {
return answers{}, err
}
+ // And why any machine cannot be worked out at all, which is neither of the first two questions
+ // and is asked before them both in practice: a machine nothing can be computed for is not
+ // broken, not quiet and not behind, and every other answer here would call it well.
+ //
+ // Read through whoResolves, which is what the private network is built from, so this and the
+ // network agree about who could not be resolved rather than deciding it twice.
+ _, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
+ if err != nil {
+ return answers{}, err
+ }
+
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
// this one says a machine is — and only the second has anybody's change waiting in it.
+ //
+ // **One machine that cannot be resolved must not take the answer away from every other**
+ // (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
+ // is the blocked machine has no hub — which used to come back here as a refusal, so `status`
+ // said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
+ // reason is kept and reported as data; every question that does not depend on it is still
+ // answered.
would, err := wouldSend(ctx, open, out.nodes)
if err != nil {
- return answers{}, err
+ out.network = err.Error()
+ would = map[string]string{}
}
out.waiting, err = inv.Waiting(ctx, would)
if err != nil {
diff --git a/cmd/mesh-control/status_test.go b/cmd/mesh-control/status_test.go
new file mode 100644
index 0000000..db5102c
--- /dev/null
+++ b/cmd/mesh-control/status_test.go
@@ -0,0 +1,121 @@
+package main
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+// **One machine's failure must never take the answer away from a reader asking about the others.**
+//
+// A blocked machine is not on the private network, and a mesh whose hub is that machine has no hub
+// — which came back through the reading as a refusal, so `status` printed nothing at all and
+// `status --json` emitted multi-line prose on stderr and not one byte of JSON. A machine-readable
+// interface that stops being machine-readable exactly when something is wrong is one nobody can
+// build an alarm on.
+func TestOneBlockedMachineDoesNotDestroyTheWholeDocument(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+
+ // Break the hub, using nothing but the command a person has.
+ one, two := rivals()
+ register(t, open, one)
+ register(t, open, two)
+ for _, m := range []string{"rival-one", "rival-two"} {
+ if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
+ t.Fatal(err)
+ }
+ }
+
+ asked, err := theThreeQuestions(ctx, open)
+ if err != nil {
+ t.Fatalf("one blocked machine made the whole mesh unreadable: %v", err)
+ }
+ body, err := statusAsJSON(asked)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ t.Fatalf("what a script would read is not JSON: %v\n%s", err, body)
+ }
+
+ // The failure is in the document, as data, on the machine it belongs to.
+ unresolved, _ := parsed["unresolved"].([]any)
+ if len(unresolved) == 0 {
+ t.Fatalf("a machine that cannot be worked out is absent from the document:\n%s", body)
+ }
+ var found bool
+ for _, row := range unresolved {
+ entry, _ := row.(map[string]any)
+ if entry["node"] != "anchor" {
+ continue
+ }
+ found = true
+ problem, _ := entry["problem"].(string)
+ if !strings.Contains(problem, "the-seat") {
+ t.Errorf("the machine's problem is not its own words: %q", problem)
+ }
+ }
+ if !found {
+ t.Fatalf("the blocked machine is not the one named:\n%s", body)
+ }
+ // And the mesh is still counted, so a reader can tell "none blocked" from "none at all".
+ if parsed["machines"] != float64(2) {
+ t.Errorf("the rest of the document did not survive: %v", parsed["machines"])
+ }
+}
+
+// A well mesh carries the field as an empty list, not as nothing: a reader distinguishing "none
+// blocked" from "this field is missing" would have to handle both, and null is the one that gets
+// forgotten.
+func TestAWellMeshCarriesAnEmptyUnresolvedList(t *testing.T) {
+ open := aMesh(t)
+ asked, err := theThreeQuestions(t.Context(), open)
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := statusAsJSON(asked)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ t.Fatal(err)
+ }
+ list, ok := parsed["unresolved"].([]any)
+ if !ok {
+ t.Fatalf("\"unresolved\" is %T, not a list:\n%s", parsed["unresolved"], body)
+ }
+ if len(list) != 0 {
+ t.Fatalf("a well mesh reports blocked machines: %v", list)
+ }
+ if _, said := parsed["network"]; said {
+ t.Errorf("a well mesh says the network could not be computed: %v", parsed["network"])
+ }
+}
+
+// And the page says it too, from the same reading. A board that renders "all well" over a mesh
+// where nothing can be sent anywhere is worse than a board that is down.
+func TestThePageSaysWhichMachinesCannotBeWorkedOut(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ one, two := rivals()
+ register(t, open, one)
+ register(t, open, two)
+ for _, m := range []string{"rival-one", "rival-two"} {
+ if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
+ t.Fatal(err)
+ }
+ }
+ asked, err := theThreeQuestions(ctx, open)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rendered := render(t, viewOf(asked))
+ for _, want := range []string{"Can everything be worked out?", "anchor", "the-seat"} {
+ if !strings.Contains(rendered, want) {
+ t.Fatalf("the page does not say %q:\n%s", want, rendered)
+ }
+ }
+}