diff --git a/cmd/mesh-control/acts.go b/cmd/mesh-control/acts.go
index 0586ceb..7911024 100644
--- a/cmd/mesh-control/acts.go
+++ b/cmd/mesh-control/acts.go
@@ -3,6 +3,8 @@ package main
import (
"context"
"fmt"
+ "sort"
+ "strings"
"github.com/novox/mesh-control/internal/catalogue"
)
@@ -17,11 +19,25 @@ import (
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
// composes its own explanation, because two explanations of one refusal drift.
-// assign puts a module on a node, and says at once whether the whole set still resolves.
+// assign puts a module on a node, and says at once what in the mesh no longer works out.
//
-// The assignment is kept even when it does not: it is what a person meant, and the refusal is
-// about the set rather than about this one. That is a decision, so it lives here rather than in
-// whichever surface asked.
+// The assignment is kept even when the node does not resolve: it is what a person meant, and
+// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
+// long as it takes to assign the provider, and refusing the first half of a pair would make the
+// order somebody types two commands in part of the mesh's rules.
+//
+// **What is checked is the mesh, not the one machine** — because that is what the assignment
+// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
+// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
+// action tested one node and the verify is resolution over all of them, so an assignment could be
+// reported as fine while it took a provision away from every other machine — a module offering a
+// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
+// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
+// module already assigned. That was found on a four-node raise and read as a version bump breaking
+// provider recognition; it was neither the version nor the provider.
+//
+// It costs a resolution per machine. Assignment is a person typing a command, and being told which
+// machines this just blocked is worth more than the milliseconds.
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
if err := open.inventory.Assign(ctx, node, module); err != nil {
return "", err
@@ -29,8 +45,9 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
said := fmt.Sprintf("%s is assigned %s", node, module)
plan, _, err := planFor(ctx, open, node)
if err != nil {
- // Kept, and still refused. Both halves are the answer.
- return said, err
+ // Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
+ // worth reporting: this machine's refusal is rarely the only consequence.
+ return said + blockedElsewhere(ctx, open, node), err
}
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
// capability the machine lacks is on the wrong machine, and the assignment records what a person
@@ -43,15 +60,66 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
}
}
- return said + fmt.Sprintf("\n run `push %s` to send it", node), nil
+ return said + fmt.Sprintf("\n run `push %s` to send it", node) +
+ blockedElsewhere(ctx, open, node), nil
}
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
+//
+// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
+// module off one machine is the ordinary way to stop providing something to another, and nothing
+// about the command's own output would ever have said so.
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
if err := open.inventory.Unassign(ctx, node, module); err != nil {
return "", err
}
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
- node, module, node), nil
+ node, module, node) + blockedElsewhere(ctx, open, node), nil
+}
+
+// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
+//
+// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
+// whole mesh twice and would still be a guess about which of several changes did it; saying
+// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
+// and cannot mislead. The machine that was just changed is left out because its own refusal is
+// already the answer beside this one.
+//
+// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
+// not a reason to claim the assignment did not happen — it did.
+func blockedElsewhere(ctx context.Context, open *stores, except string) string {
+ nodes, err := open.inventory.Nodes(ctx)
+ if err != nil {
+ return "\n\nThe rest of the mesh could not be checked: " + err.Error()
+ }
+ blocked := map[string]string{}
+ for _, n := range nodes {
+ if n.Name == except {
+ continue
+ }
+ if _, _, err := planFor(ctx, open, n.Name); err != nil {
+ blocked[n.Name] = err.Error()
+ }
+ }
+ if len(blocked) == 0 {
+ return ""
+ }
+ names := make([]string, 0, len(blocked))
+ for name := range blocked {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+
+ var out strings.Builder
+ fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
+ "nothing will be sent to them:\n", len(names))
+ for _, name := range names {
+ fmt.Fprintf(&out, " %s\n", name)
+ for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
+ fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
+ }
+ }
+ out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
+ return out.String()
}
diff --git a/cmd/mesh-control/acts_test.go b/cmd/mesh-control/acts_test.go
new file mode 100644
index 0000000..7cbc642
--- /dev/null
+++ b/cmd/mesh-control/acts_test.go
@@ -0,0 +1,133 @@
+package main
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/novox/mesh-control/internal/catalogue"
+)
+
+// What an assignment says about the machines it was not about.
+
+// **An assignment that blocks another machine says so.**
+//
+// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it
+// the moment its own node stops resolving, so an assignment to the provider's machine silently took
+// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this
+// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the
+// way back, as a version bump breaking provider recognition. It was neither the version nor the
+// provider: it was one machine's set of assignments, and nothing said so.
+//
+// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify
+// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the
+// verify is resolution over all of them.
+func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+
+ // A provider on the hub and a consumer on the other machine, both resolving.
+ register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
+ Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
+ Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
+ register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
+ Requires: []string{"acme-ca"}})
+ if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
+ t.Fatal(err)
+ }
+ said, err := assign(ctx, open, "laptop", "route-proxy")
+ if err != nil {
+ t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said)
+ }
+ if strings.Contains(said, "cannot be worked out") {
+ t.Fatalf("a well mesh was reported as blocked:\n%s", said)
+ }
+
+ // Now break the hub's own set, with nothing but the command a person has.
+ one, two := rivals()
+ register(t, open, one)
+ register(t, open, two)
+ if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil {
+ t.Fatal(err)
+ }
+ said, err = assign(ctx, open, "anchor", "rival-two")
+ if err == nil {
+ t.Fatal("an assignment that makes its own node incoherent was not reported at all")
+ }
+
+ // The node's own refusal is the error, as it always was. What is new is that the machines this
+ // just took a provision away from are named in the same breath.
+ if !strings.Contains(said, "laptop") {
+ t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err)
+ }
+ if !strings.Contains(said, "acme-ca") {
+ t.Fatalf("what laptop is now missing is not said:\n%s", said)
+ }
+ if !strings.Contains(said, "cannot be worked out as things stand") {
+ t.Fatalf("the report does not say what state the mesh is in:\n%s", said)
+ }
+ // And the assignment is kept: it is what a person meant, and assignment is not an ordering.
+ assigned, err := open.inventory.Assigned(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !contains(assigned, "rival-two") {
+ t.Fatalf("the assignment was not kept: %v", assigned)
+ }
+}
+
+// A consumer assigned before its provider is refused for itself and kept, because assignment is not
+// an ordering — refusing the first half of a pair would make the order somebody types two commands
+// in part of the mesh's rules.
+func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
+ Requires: []string{"acme-ca"}})
+
+ said, err := assign(ctx, open, "laptop", "route-proxy")
+ if err == nil {
+ t.Fatalf("a consumer with nothing to consume resolved:\n%s", said)
+ }
+ assigned, err := open.inventory.Assigned(ctx, "laptop")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !contains(assigned, "route-proxy") {
+ t.Fatalf("assignment became an ordering: %v", assigned)
+ }
+}
+
+// Unassigning is the ordinary way to stop providing something to another machine, and it reports
+// the same way for the same reason.
+func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
+ Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
+ Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
+ register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
+ Requires: []string{"acme-ca"}})
+ if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
+ t.Fatal(err)
+ }
+
+ said, err := unassign(ctx, open, "anchor", "step-ca")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") {
+ t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said)
+ }
+}
+
+func contains(all []string, one string) bool {
+ for _, s := range all {
+ if s == one {
+ return true
+ }
+ }
+ return false
+}
diff --git a/cmd/mesh-control/board.go b/cmd/mesh-control/board.go
index fc11d2a..49702ac 100644
--- a/cmd/mesh-control/board.go
+++ b/cmd/mesh-control/board.go
@@ -7,6 +7,8 @@ import (
"fmt"
"html/template"
"net/http"
+ "sort"
+ "strings"
"time"
)
@@ -94,8 +96,7 @@ func board() http.Handler {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
- body, err := statusAsJSON(asked.wrong, asked.nodes, asked.quiet, asked.behind, asked.sources,
- asked.waiting, asked.reported)
+ body, err := statusAsJSON(asked)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -129,7 +130,21 @@ type view struct {
Quiet []quietMachine
Behind []staleModule
Waiting []waitingMachine
- At string
+ // Unresolved is every machine that cannot be worked out at all. Shown above everything else,
+ // because a machine here is in none of the other lists: nothing was computed for it, so it is
+ // not broken, not quiet and not behind — and a page without this said "all well" about a mesh
+ // where nothing could be sent anywhere.
+ Unresolved []blockedMachine
+ // Network is why the private network could not be computed, when it could not.
+ Network string
+ At string
+}
+
+type blockedMachine struct {
+ Node string
+ // Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
+ // a machine is usually blocked by more than one and fixing one of them changes nothing.
+ Said []string
}
type waitingMachine struct {
@@ -165,7 +180,20 @@ type staleModule struct {
}
func viewOf(asked answers) view {
- out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05")}
+ out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
+ Network: asked.network}
+ var blocked []string
+ for name := range asked.refused {
+ blocked = append(blocked, name)
+ }
+ sort.Strings(blocked)
+ for _, name := range blocked {
+ one := blockedMachine{Node: name}
+ for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
+ one.Said = append(one.Said, strings.TrimSpace(line))
+ }
+ out.Unresolved = append(out.Unresolved, one)
+ }
for _, d := range asked.wrong {
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
When: d.At.Local().Format("2006-01-02 15:04")}
@@ -223,6 +251,22 @@ find out.
{{else}}
{{.Machines}} machine{{if ne .Machines 1}}s{{end}}
+{{if .Unresolved}}
+Can everything be worked out?
+
+ {{range .Unresolved}}
+ - blocked {{.Node}}
+ {{range .Said}}
{{.}}
{{end}}
+
+ {{end}}
+
+Nothing can be sent to a machine here, and it appears in none of the lists below:
+nothing was computed for it, so there is nothing it can be behind.
+{{end}}
+{{if .Network}}
+The private network could not be computed: {{.Network}}
+{{end}}
+
Is anything broken?
{{if .Broken}}
diff --git a/cmd/mesh-control/build.go b/cmd/mesh-control/build.go
index 6c7d9b8..4204f39 100644
--- a/cmd/mesh-control/build.go
+++ b/cmd/mesh-control/build.go
@@ -442,4 +442,13 @@ type answers struct {
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
// recorded at send, not at apply).
reported []inventory.Reported
+ // refused is why a machine cannot be worked out at all, by name. A different thing from every
+ // other answer here: those are about a machine that was told something, and this is about one
+ // that cannot be told anything — it never reaches waiting, because nothing was computed for it
+ // to compare against, so without this a wholly blocked mesh reads as a well one.
+ refused map[string]string
+ // network is why the private network could not be computed, when it could not. Almost always
+ // a consequence of the refusals above: a node that does not resolve is not on the network, and
+ // a mesh whose hub is that node has no hub.
+ network string
}
diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go
index d3c4d49..d40d62b 100644
--- a/cmd/mesh-control/main.go
+++ b/cmd/mesh-control/main.go
@@ -123,6 +123,9 @@ func usage() {
node add create a node record
node list the nodes this mesh knows about
node show what one machine reported it can do, and why
+ node public-domain the domain it composes its routed names under
+ node public-domain ...set it to d
+ node public-domain --clear ...it faces the outside no longer
token issue --node a one-time right to join, for an existing record
token issue --new create the record and issue for it
identity show this control plane's signing key
@@ -134,7 +137,8 @@ func usage() {
module add register a module from its manifest
module list what modules this mesh knows about
module moved the source has a newer commit than the mesh built
- module forget remove one, unless a node is running it
+ module forget remove one, unless a node runs it or the mesh holds things for it
+ module forget --and-what-it-holds ...and discard its settings, secrets and ports too
module issue --node a broker account for a module, scoped to its emits and consumes
status [--json] what is wrong, what is quiet, and what is out of date
board [--listen ADDR] the same three questions, as a page that holds nothing
diff --git a/cmd/mesh-control/mesh_for_test.go b/cmd/mesh-control/mesh_for_test.go
new file mode 100644
index 0000000..43ff46f
--- /dev/null
+++ b/cmd/mesh-control/mesh_for_test.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+ "crypto/ecdh"
+ "crypto/rand"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "testing"
+
+ "github.com/novox/mesh-control/internal/catalogue"
+ "github.com/novox/mesh-control/internal/inventory"
+ "github.com/novox/mesh-control/internal/licences"
+ "github.com/novox/mesh-control/internal/overlay"
+)
+
+// A mesh a command can be run against.
+//
+// The commands here were tested through the pieces they call and never through themselves, so
+// three faults that only exist where the pieces meet — an assignment reported as fine while it
+// blocked other machines, a read-shaped invocation that wrote, a JSON interface that stopped
+// emitting JSON — were invisible to every test in this package. This raises the real stores and
+// calls the real functions.
+
+// aMesh is two placed, capable machines on a private network, with nothing assigned but the
+// network itself.
+//
+// `anchor` is the hub. That is not decoration: a mesh whose hub cannot be resolved has no private
+// network at all, which is how one machine's problem reaches every other.
+func aMesh(t *testing.T) *stores {
+ t.Helper()
+ inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
+ licences.ForTest(t) // planning reaches this one too, by name and never by connection
+
+ open, err := openStores(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(open.Close)
+ for _, m := range provided {
+ if err := open.inventory.Provide(t.Context(), m); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for i, name := range []string{"anchor", "laptop"} {
+ record, err := open.inventory.AddNode(t.Context(), name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := open.inventory.SetPlace(t.Context(), name, name+".example:51820", "here",
+ name == "anchor", fmt.Sprintf("10.77.0.%d", i+1)); err != nil {
+ t.Fatal(err)
+ }
+ reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
+ {"name": "container-runtime", "present": true},
+ {"name": "wireguard", "present": true},
+ {"name": "systemd", "present": true},
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var profile map[string]any
+ if err := json.Unmarshal(reported, &profile); err != nil {
+ t.Fatal(err)
+ }
+ if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
+ t.Fatal(err)
+ }
+ if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
+ t.Fatal(err)
+ }
+ if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
+ t.Fatal(err)
+ }
+ if err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return open
+}
+
+// aPublicKey is a key a machine could have reported. Its private half is thrown away: nothing here
+// opens anything, it only needs the mesh to believe a machine has a key.
+func aPublicKey(t *testing.T) string {
+ t.Helper()
+ k, err := ecdh.X25519().GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
+}
+
+// register puts a manifest in the catalogue.
+func register(t *testing.T, open *stores, m catalogue.Manifest) {
+ t.Helper()
+ if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// rivals are two modules that cannot share a machine, which is the shortest way to make a node's
+// own set of assignments incoherent using nothing but commands a person has.
+func rivals() (catalogue.Manifest, catalogue.Manifest) {
+ claim := []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}
+ return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
+ catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
+}
diff --git a/cmd/mesh-control/modules.go b/cmd/mesh-control/modules.go
index c5e4f46..96e2c3b 100644
--- a/cmd/mesh-control/modules.go
+++ b/cmd/mesh-control/modules.go
@@ -188,13 +188,37 @@ func moduleCommand(ctx context.Context, args []string) error {
return nil
case "forget":
- if len(args) != 2 {
- return errors.New("module forget ")
- }
- if err := inv.ForgetModule(ctx, args[1]); err != nil {
+ // **What goes with it is said before it goes** (novox/hq 04-ISSUES/017). The settings, the
+ // module's own secrets and the ports the mesh chose all cascade off the module row, so
+ // `forget` used to destroy them and report "forgotten" — an action succeeding into a state
+ // its own verify would reject, and a sealed secret is not recoverable afterwards.
+ set := flag.NewFlagSet("module forget", flag.ContinueOnError)
+ andHeld := set.Bool("and-what-it-holds", false,
+ "discard its settings, its own secrets and its ports along with it")
+ positionals, err := parseAround(set, args[1:])
+ if err != nil {
return err
}
- fmt.Printf("%s forgotten\n", args[1])
+ if len(positionals) != 1 {
+ return errors.New("module forget [--and-what-it-holds]")
+ }
+ if !*andHeld {
+ if err := inv.ForgetModule(ctx, positionals[0]); err != nil {
+ return err
+ }
+ fmt.Printf("%s forgotten\n", positionals[0])
+ return nil
+ }
+ held, err := inv.DiscardModule(ctx, positionals[0])
+ if err != nil {
+ return err
+ }
+ fmt.Printf("%s forgotten\n", positionals[0])
+ for _, line := range held.Lines() {
+ // Said after the fact as well as before it: this is the only record that these
+ // existed, and the next person to ask why the module came back empty reads it here.
+ fmt.Printf(" discarded%s\n", strings.TrimPrefix(line, " "))
+ }
return nil
case "issue":
diff --git a/cmd/mesh-control/network.go b/cmd/mesh-control/network.go
index 53ce35c..c1a2d82 100644
--- a/cmd/mesh-control/network.go
+++ b/cmd/mesh-control/network.go
@@ -59,7 +59,8 @@ func overlayCommand(ctx context.Context, args []string) error {
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
if len(args) == 0 {
- return errors.New("overlay place [--endpoint host:port] [--site name] [--hub]")
+ return errors.New(
+ "overlay place [--endpoint host:port] [--site name] [--hub], or --nothing")
}
node := args[0]
@@ -67,10 +68,31 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
site := set.String("site", "", "where this machine physically is, or empty if it roams")
hub := set.Bool("hub", false, "this node is the hub every other routes through")
+ nothing := set.Bool("nothing", false,
+ "place it with nothing set: not dialable, no site, not the hub")
if err := set.Parse(args[1:]); err != nil {
return err
}
+ // **All three are declared together, so saying nothing took all three away.** The sibling of
+ // `node public-domain`: `overlay place anchor` reads like it places the node it names, and it
+ // silently unset the endpoint every other machine dials, the site it is in, and the hub if it
+ // was the hub — every path through it going with them, at the moment somebody was trying to
+ // look at it.
+ //
+ // A placement with nothing set is a real thing to want — a machine that roams and opens every
+ // path itself is exactly that — so it keeps a way to say so, by name.
+ if set.NFlag() == 0 {
+ return fmt.Errorf("overlay place %s was given nothing to place it with, and all three are "+
+ "declared together — it would take away the endpoint other machines dial %s at, its "+
+ "site, and the hub if it is the hub. Say --endpoint/--site/--hub, or --nothing if that "+
+ "is what you meant", node, node)
+ }
+ if *nothing && (*endpoint != "" || *site != "" || *hub) {
+ return fmt.Errorf("give %s a placement or --nothing, not both: they say opposite things "+
+ "and the mesh will not choose between them", node)
+ }
+
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
// election by address prefix fails silently (novox/hq ADR 0007).
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
diff --git a/cmd/mesh-control/network_test.go b/cmd/mesh-control/network_test.go
new file mode 100644
index 0000000..3c354c6
--- /dev/null
+++ b/cmd/mesh-control/network_test.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/novox/mesh-control/internal/inventory"
+)
+
+// placementOf is what the mesh holds about where one node is.
+func placementOf(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string) inventory.Overlay {
+ t.Helper()
+ placed, err := inv.Overlays(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, one := range placed {
+ if one.Name == name {
+ return one
+ }
+ }
+ t.Fatalf("%s is not placed at all", name)
+ return inventory.Overlay{}
+}
+
+// The sibling of `node public-domain`, and the worse one: a placement is three facts declared
+// together, so an invocation that said none of them took all three away — the endpoint every other
+// machine dials, the site, and the hub. A mesh whose hub was placed that way has no paths left.
+func TestPlacingANodeWithNothingSaidDoesNotUnplaceIt(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := overlayPlace(ctx, open.inventory,
+ []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"}); err != nil {
+ t.Fatal(err)
+ }
+
+ err := overlayPlace(ctx, open.inventory, []string{"anchor"})
+ if err == nil {
+ t.Fatal("saying nothing unplaced the node instead of being refused")
+ }
+ if !strings.Contains(err.Error(), "--nothing") {
+ t.Errorf("the refusal does not say how to mean it: %v", err)
+ }
+
+ held := placementOf(t, ctx, open.inventory, "anchor")
+ if held.Endpoint != "198.51.100.10:51820" || held.Site != "hosting" || !held.Hub {
+ t.Fatalf("the placement was taken away by an invocation that was refused: %+v", held)
+ }
+}
+
+// Placing a machine with nothing set is a real thing to want — one that roams and opens every path
+// itself is exactly that — so it keeps a way to be said, by name.
+func TestPlacingANodeWithNothingIsAskedForByName(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := overlayPlace(ctx, open.inventory,
+ []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := overlayPlace(ctx, open.inventory, []string{"anchor", "--nothing"}); err != nil {
+ t.Fatal(err)
+ }
+ held := placementOf(t, ctx, open.inventory, "anchor")
+ if held.Endpoint != "" || held.Site != "" || held.Hub {
+ t.Fatalf("--nothing did not place it with nothing: %+v", held)
+ }
+}
+
+// Both at once cannot be meant, so neither silently wins.
+func TestAPlacementAndNothingTogetherIsRefused(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := overlayPlace(ctx, open.inventory,
+ []string{"anchor", "--site", "hosting", "--nothing"}); err == nil {
+ t.Fatal("a placement and --nothing together was accepted")
+ }
+}
diff --git a/cmd/mesh-control/nodes.go b/cmd/mesh-control/nodes.go
index 587f147..d4265f1 100644
--- a/cmd/mesh-control/nodes.go
+++ b/cmd/mesh-control/nodes.go
@@ -22,7 +22,7 @@ import (
func nodeCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
- return errors.New("node add , node list, node show , or node public-domain [domain]")
+ return errors.New("node add , node list, node show , or " + publicDomainUsage)
}
open, err := openStores(ctx)
if err != nil {
@@ -66,32 +66,84 @@ func nodeCommand(ctx context.Context, args []string) error {
return nil
case "public-domain":
- // The domain this node composes its routed names under (novox/hq ADR 0056). Given a domain,
- // it is set; given nothing, it is cleared — a node that stops facing the outside composes no
- // names. Lab-versus-production is this one setting and nothing else (see the ADR).
- if len(args) < 2 || len(args) > 3 {
- return errors.New(
- "node public-domain [domain] — a domain sets it, nothing clears it")
- }
- domain := ""
- if len(args) == 3 {
- domain = args[2]
- }
- if err := inv.SetPublicDomain(ctx, args[1], domain); err != nil {
- return err
- }
- if domain == "" {
- fmt.Printf("%s has no public domain, so it composes no routed names\n", args[1])
- } else {
- fmt.Printf("%s composes its routed names under %s\n", args[1], domain)
- }
- return nil
+ // The domain this node composes its routed names under (novox/hq ADR 0056).
+ //
+ // **The form with no argument reports; clearing is asked for by name.** It used to clear —
+ // so `node public-domain anchor`, which reads like a question and is what anybody types to
+ // find out what the answer is, silently took every routed name the node had. A read-shaped
+ // invocation must never be a destructive write: there is no output that makes up for it,
+ // because the damage is already done by the time it prints.
+ return publicDomain(ctx, inv, args[1:])
default:
return fmt.Errorf("node has no %q; it has add, list, show and public-domain", args[0])
}
}
+// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
+const publicDomainUsage = "node public-domain — what it is now; " +
+ " to set it; --clear to take it away"
+
+// publicDomain reads, sets or clears the domain a node composes its routed names under.
+//
+// Three forms, and the destructive one is the only one that has to be asked for. Clearing is a
+// real thing to want — a machine that stops facing the outside composes no names, and
+// lab-versus-production is this one setting (novox/hq ADR 0056) — so it keeps a way to say it.
+// What it does not keep is being the thing that happens when nothing was said at all.
+func publicDomain(ctx context.Context, inv *inventory.Inventory, args []string) error {
+ set := flag.NewFlagSet("node public-domain", flag.ContinueOnError)
+ clear := set.Bool("clear", false, "take the domain away; it composes no routed names after")
+ positionals, err := parseAround(set, args)
+ if err != nil {
+ return err
+ }
+ if len(positionals) == 0 || len(positionals) > 2 {
+ return errors.New(publicDomainUsage)
+ }
+ node := positionals[0]
+
+ switch {
+ case *clear && len(positionals) == 2:
+ // Both, which cannot be meant. Refused rather than one of them silently winning.
+ return fmt.Errorf("give %s a domain or --clear, not both: %q and --clear say opposite "+
+ "things and the mesh will not choose between them", node, positionals[1])
+
+ case *clear:
+ if err := inv.SetPublicDomain(ctx, node, ""); err != nil {
+ return err
+ }
+ fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
+ fmt.Printf(" run `push %s` to take them off it\n", node)
+ return nil
+
+ case len(positionals) == 2:
+ if err := inv.SetPublicDomain(ctx, node, positionals[1]); err != nil {
+ return err
+ }
+ fmt.Printf("%s composes its routed names under %s\n", node, positionals[1])
+ fmt.Printf(" run `push %s` to send it\n", node)
+ return nil
+
+ default:
+ // Asked, so answered. NodeByName first, so a name the mesh has never heard of is a refusal
+ // rather than "it has no public domain", which is true of that name and says nothing.
+ if _, err := inv.NodeByName(ctx, node); err != nil {
+ return err
+ }
+ domain, err := inv.PublicDomainOf(ctx, node)
+ if err != nil {
+ return err
+ }
+ if domain == "" {
+ fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
+ fmt.Printf(" `node public-domain %s ` gives it one\n", node)
+ return nil
+ }
+ fmt.Printf("%s composes its routed names under %s\n", node, domain)
+ return nil
+ }
+}
+
func tokenCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "issue" {
return errors.New("token issue --node , or token issue --new ")
diff --git a/cmd/mesh-control/nodes_test.go b/cmd/mesh-control/nodes_test.go
new file mode 100644
index 0000000..a27d3ba
--- /dev/null
+++ b/cmd/mesh-control/nodes_test.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+ "strings"
+ "testing"
+)
+
+// **A read-shaped invocation is never a destructive write.**
+//
+// `node public-domain anchor` used to clear the domain. It reads like a question — it is what
+// anybody types to find out what the answer is — and it silently took every routed name the node
+// had. There is no output that makes up for that: by the time it prints, the fact is gone.
+func TestAskingForANodesPublicDomainDoesNotTakeItAway(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := publicDomain(ctx, open.inventory, []string{"anchor"}); err != nil {
+ t.Fatal(err)
+ }
+ domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if domain != "example.test" {
+ t.Fatalf("asking what the domain is took it away: %q", domain)
+ }
+}
+
+// Clearing is a real thing to want — a machine that stops facing the outside composes no names —
+// so it keeps a way to be said. What it stops being is what happens when nothing was said.
+func TestClearingANodesPublicDomainIsAskedForByName(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
+ t.Fatal(err)
+ }
+ if err := publicDomain(ctx, open.inventory, []string{"anchor", "--clear"}); err != nil {
+ t.Fatal(err)
+ }
+ domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if domain != "" {
+ t.Fatalf("--clear did not clear it: %q", domain)
+ }
+}
+
+// A domain sets it, as it always did.
+func TestGivingANodeAPublicDomainSetsIt(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := publicDomain(ctx, open.inventory, []string{"anchor", "example.test"}); err != nil {
+ t.Fatal(err)
+ }
+ domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if domain != "example.test" {
+ t.Fatalf("got %q", domain)
+ }
+}
+
+// A domain and --clear say opposite things. Refused rather than one of them silently winning.
+func TestADomainAndClearTogetherIsRefused(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
+ t.Fatal(err)
+ }
+ err := publicDomain(ctx, open.inventory, []string{"anchor", "other.test", "--clear"})
+ if err == nil {
+ t.Fatal("a domain and --clear together were accepted")
+ }
+ if !strings.Contains(err.Error(), "not both") {
+ t.Fatalf("the refusal does not say why: %v", err)
+ }
+ // And neither half happened.
+ domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if domain != "example.test" {
+ t.Fatalf("a refused command changed something: %q", domain)
+ }
+}
+
+// Asking about a name the mesh has never heard of is a refusal, not "it has no public domain" —
+// which is true of that name and says nothing.
+func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
+ open := aMesh(t)
+ if err := publicDomain(t.Context(), open.inventory, []string{"nowhere"}); err == nil {
+ t.Fatal("a name the mesh does not know was answered as if it were a machine")
+ }
+}
diff --git a/cmd/mesh-control/readable.go b/cmd/mesh-control/readable.go
index 346e93c..68c8bc6 100644
--- a/cmd/mesh-control/readable.go
+++ b/cmd/mesh-control/readable.go
@@ -3,9 +3,8 @@ package main
import (
"encoding/json"
"fmt"
+ "sort"
"time"
-
- "github.com/novox/mesh-control/internal/inventory"
)
// The same answers, in a shape something other than a person can read.
@@ -40,11 +39,30 @@ type meshStatus struct {
// whose is older is still working — and Waiting cannot tell those apart, because the sent
// digest is recorded at send, not at apply.
Reported []machineReported `json:"reported"`
+ // Unresolved is every machine that cannot be worked out at all, with what the mesh said when
+ // it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
+ // there is nothing to compare it against and nothing it can be behind — which is why a
+ // document without this field described a wholly blocked mesh as a well one.
+ //
+ // Per machine, and data. One node failing must never take the document away from a reader
+ // asking about the others.
+ Unresolved []machineUnresolved `json:"unresolved"`
+ // Network is why the private network could not be computed, when it could not; absent when it
+ // could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
+ // network, and a mesh whose hub is that node has no hub.
+ Network string `json:"network,omitempty"`
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
// "none at all".
Machines int `json:"machines"`
}
+type machineUnresolved struct {
+ Node string `json:"node"`
+ // Problem is the mesh's own words, whole — newlines and all. It lists every requirement that
+ // could not be met, and a first line alone would name one of them and hide the rest.
+ Problem string `json:"problem"`
+}
+
type machineDoing struct {
Node string `json:"node"`
// Outcome is refused or failed. Kept distinct all the way out: they are fixed in different
@@ -92,14 +110,32 @@ type moduleBehind struct {
On []string `json:"on"`
}
-// statusAsJSON answers the same three questions as the text form, from the same calls.
-func statusAsJSON(wrong []inventory.Doing, nodes []inventory.Node, quiet []inventory.Node,
- behind map[string][]string, sources map[string]inventory.Source,
- waiting []inventory.Machine, reported []inventory.Reported) ([]byte, error) {
+// statusAsJSON answers the same questions as the text form, from the same reading.
+//
+// **It takes the whole reading rather than a growing argument list**, which is what let a new
+// answer be added to the text form and forgotten here — the two are one function's output in two
+// shapes, and they must not be able to differ about what was asked.
+//
+// It never fails on account of the mesh. Every per-machine problem in here is a field, so one
+// machine that cannot be worked out cannot stop a caller reading about the others: a
+// machine-readable interface that stops being machine-readable exactly when something is wrong is
+// one nobody can build an alarm on.
+func statusAsJSON(asked answers) ([]byte, error) {
+ wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
+ behind, sources := asked.behind, asked.sources
+ waiting, reported := asked.waiting, asked.reported
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
- Reported: []machineReported{}}
+ Reported: []machineReported{}, Unresolved: []machineUnresolved{},
+ Network: asked.network}
+ for name := range asked.refused {
+ out.Unresolved = append(out.Unresolved, machineUnresolved{
+ Node: name, Problem: asked.refused[name]})
+ }
+ sort.Slice(out.Unresolved, func(i, j int) bool {
+ return out.Unresolved[i].Node < out.Unresolved[j].Node
+ })
for _, r := range reported {
out.Reported = append(out.Reported, machineReported{
Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
diff --git a/cmd/mesh-control/readable_test.go b/cmd/mesh-control/readable_test.go
index 3d1504f..2091d2a 100644
--- a/cmd/mesh-control/readable_test.go
+++ b/cmd/mesh-control/readable_test.go
@@ -17,7 +17,8 @@ import (
func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
behind map[string][]string, sources map[string]inventory.Source) map[string]any {
t.Helper()
- body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, nil, nil)
+ body, err := statusAsJSON(answers{
+ wrong: wrong, nodes: nodes, quiet: quiet, behind: behind, sources: sources})
if err != nil {
t.Fatal(err)
}
@@ -123,10 +124,12 @@ func TestNoSecretIsInWhatABoardReads(t *testing.T) {
// Everything here comes from the mesh's own records, which hold no readable secret — but a
// shape a page is built against is exactly where one would eventually be added for
// convenience, so this says it out loud.
- body, err := statusAsJSON(
- []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
+ body, err := statusAsJSON(answers{
+ wrong: []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
Refused: "resource \"x\": a file needs a path"}},
- []inventory.Node{{Name: "a"}}, nil, nil, nil, nil, nil)
+ nodes: []inventory.Node{{Name: "a"}},
+ refused: map[string]string{"a": "nothing provides \"database\", wanted by web"},
+ })
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/mesh-control/status.go b/cmd/mesh-control/status.go
index 6c3bf49..86bd83a 100644
--- a/cmd/mesh-control/status.go
+++ b/cmd/mesh-control/status.go
@@ -9,6 +9,7 @@ import (
"time"
"github.com/novox/mesh-control/internal/inventory"
+ "github.com/novox/mesh-control/internal/overlay"
)
// is anything broken, is anything not answering, is anything out of date.
@@ -54,8 +55,7 @@ func statusCommand(ctx context.Context, args []string) error {
behind, sources := asked.behind, asked.sources
if *asJSON {
- body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting,
- asked.reported)
+ body, err := statusAsJSON(asked)
if err != nil {
return err
}
@@ -63,6 +63,30 @@ func statusCommand(ctx context.Context, args []string) error {
return nil
}
+ if len(asked.refused) > 0 {
+ // First, above everything else. A machine that cannot be worked out is not running an old
+ // declaration — it has no declaration, and nothing below this line is about it.
+ var names []string
+ for name := range asked.refused {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
+ len(names))
+ for _, name := range names {
+ fmt.Printf(" %s\n", name)
+ for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
+ fmt.Printf(" %s\n", strings.TrimSpace(line))
+ }
+ }
+ fmt.Println()
+ }
+
+ if asked.network != "" {
+ fmt.Printf("the private network could not be computed:\n %s\n\n",
+ strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
+ }
+
if len(wrong) > 0 {
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
for _, d := range wrong {
@@ -139,7 +163,8 @@ func statusCommand(ctx context.Context, args []string) error {
fmt.Printf("\n `push --behind` sends them\n\n")
}
- if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 {
+ if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
+ len(asked.refused) == 0 && asked.network == "" {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
@@ -197,12 +222,31 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
if err != nil {
return answers{}, err
}
+ // And why any machine cannot be worked out at all, which is neither of the first two questions
+ // and is asked before them both in practice: a machine nothing can be computed for is not
+ // broken, not quiet and not behind, and every other answer here would call it well.
+ //
+ // Read through whoResolves, which is what the private network is built from, so this and the
+ // network agree about who could not be resolved rather than deciding it twice.
+ _, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
+ if err != nil {
+ return answers{}, err
+ }
+
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
// this one says a machine is — and only the second has anybody's change waiting in it.
+ //
+ // **One machine that cannot be resolved must not take the answer away from every other**
+ // (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
+ // is the blocked machine has no hub — which used to come back here as a refusal, so `status`
+ // said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
+ // reason is kept and reported as data; every question that does not depend on it is still
+ // answered.
would, err := wouldSend(ctx, open, out.nodes)
if err != nil {
- return answers{}, err
+ out.network = err.Error()
+ would = map[string]string{}
}
out.waiting, err = inv.Waiting(ctx, would)
if err != nil {
diff --git a/cmd/mesh-control/status_test.go b/cmd/mesh-control/status_test.go
new file mode 100644
index 0000000..db5102c
--- /dev/null
+++ b/cmd/mesh-control/status_test.go
@@ -0,0 +1,121 @@
+package main
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+// **One machine's failure must never take the answer away from a reader asking about the others.**
+//
+// A blocked machine is not on the private network, and a mesh whose hub is that machine has no hub
+// — which came back through the reading as a refusal, so `status` printed nothing at all and
+// `status --json` emitted multi-line prose on stderr and not one byte of JSON. A machine-readable
+// interface that stops being machine-readable exactly when something is wrong is one nobody can
+// build an alarm on.
+func TestOneBlockedMachineDoesNotDestroyTheWholeDocument(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+
+ // Break the hub, using nothing but the command a person has.
+ one, two := rivals()
+ register(t, open, one)
+ register(t, open, two)
+ for _, m := range []string{"rival-one", "rival-two"} {
+ if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
+ t.Fatal(err)
+ }
+ }
+
+ asked, err := theThreeQuestions(ctx, open)
+ if err != nil {
+ t.Fatalf("one blocked machine made the whole mesh unreadable: %v", err)
+ }
+ body, err := statusAsJSON(asked)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ t.Fatalf("what a script would read is not JSON: %v\n%s", err, body)
+ }
+
+ // The failure is in the document, as data, on the machine it belongs to.
+ unresolved, _ := parsed["unresolved"].([]any)
+ if len(unresolved) == 0 {
+ t.Fatalf("a machine that cannot be worked out is absent from the document:\n%s", body)
+ }
+ var found bool
+ for _, row := range unresolved {
+ entry, _ := row.(map[string]any)
+ if entry["node"] != "anchor" {
+ continue
+ }
+ found = true
+ problem, _ := entry["problem"].(string)
+ if !strings.Contains(problem, "the-seat") {
+ t.Errorf("the machine's problem is not its own words: %q", problem)
+ }
+ }
+ if !found {
+ t.Fatalf("the blocked machine is not the one named:\n%s", body)
+ }
+ // And the mesh is still counted, so a reader can tell "none blocked" from "none at all".
+ if parsed["machines"] != float64(2) {
+ t.Errorf("the rest of the document did not survive: %v", parsed["machines"])
+ }
+}
+
+// A well mesh carries the field as an empty list, not as nothing: a reader distinguishing "none
+// blocked" from "this field is missing" would have to handle both, and null is the one that gets
+// forgotten.
+func TestAWellMeshCarriesAnEmptyUnresolvedList(t *testing.T) {
+ open := aMesh(t)
+ asked, err := theThreeQuestions(t.Context(), open)
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := statusAsJSON(asked)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ t.Fatal(err)
+ }
+ list, ok := parsed["unresolved"].([]any)
+ if !ok {
+ t.Fatalf("\"unresolved\" is %T, not a list:\n%s", parsed["unresolved"], body)
+ }
+ if len(list) != 0 {
+ t.Fatalf("a well mesh reports blocked machines: %v", list)
+ }
+ if _, said := parsed["network"]; said {
+ t.Errorf("a well mesh says the network could not be computed: %v", parsed["network"])
+ }
+}
+
+// And the page says it too, from the same reading. A board that renders "all well" over a mesh
+// where nothing can be sent anywhere is worse than a board that is down.
+func TestThePageSaysWhichMachinesCannotBeWorkedOut(t *testing.T) {
+ open := aMesh(t)
+ ctx := t.Context()
+ one, two := rivals()
+ register(t, open, one)
+ register(t, open, two)
+ for _, m := range []string{"rival-one", "rival-two"} {
+ if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
+ t.Fatal(err)
+ }
+ }
+ asked, err := theThreeQuestions(ctx, open)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rendered := render(t, viewOf(asked))
+ for _, want := range []string{"Can everything be worked out?", "anchor", "the-seat"} {
+ if !strings.Contains(rendered, want) {
+ t.Fatalf("the page does not say %q:\n%s", want, rendered)
+ }
+ }
+}
diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go
index 39a849a..04e6e2f 100644
--- a/internal/inventory/catalogue.go
+++ b/internal/inventory/catalogue.go
@@ -204,10 +204,165 @@ func (i *Inventory) Provided(ctx context.Context, name string) (bool, error) {
return source != nil && *source == "the control plane", nil
}
-// ForgetModule removes a module, unless a machine is running it, and never one the control plane
-// provides.
+// ErrStillHolds is why a module cannot be forgotten without saying so first.
+//
+// Its own error because it is not a fault either: the operator settings, the module's own secrets
+// and the ports the mesh chose for it are all keyed on the module by name and all cascade when the
+// row goes. Removing the module removes them, silently, and none of them can be recovered — a
+// sealed secret least of all, because the mesh discarded the plaintext when it made it.
+var ErrStillHolds = errors.New("the mesh still holds things for that module")
+
+// Holdings is everything keyed on a module that would go with it.
+//
+// **Named one at a time rather than counted.** "3 settings" tells somebody there is something to
+// lose and not whether they can afford to lose it; "the mesh-wide layer, and anchor's" tells them
+// what to write down before they type the command again.
+type Holdings struct {
+ // Mesh is true when a mesh-wide settings layer exists for the module.
+ Mesh bool
+ // Nodes are the machines with a settings layer of their own for it, sorted.
+ Nodes []string
+ // Secrets are the module's own secrets, as " on ", sorted. These are the ones the
+ // mesh cannot make again: what is stored is sealed to a machine and the plaintext is gone.
+ Secrets []string
+ // Ports are the ports the mesh chose for it, as " on ", sorted. Made once and
+ // kept (novox/hq ADR 0038) — removing the module gives that promise up.
+ Ports []string
+}
+
+// Any reports whether removing the module would discard anything.
+func (h Holdings) Any() bool {
+ return h.Mesh || len(h.Nodes) > 0 || len(h.Secrets) > 0 || len(h.Ports) > 0
+}
+
+// Lines is what would be lost, one thing per line, for a person about to decide.
+func (h Holdings) Lines() []string {
+ var out []string
+ if h.Mesh {
+ out = append(out, " settings, for the whole mesh")
+ }
+ for _, n := range h.Nodes {
+ out = append(out, " settings, on "+n)
+ }
+ for _, s := range h.Secrets {
+ out = append(out, " its own secret "+s+" — sealed, so the mesh cannot make it again")
+ }
+ for _, p := range h.Ports {
+ out = append(out, " the port "+p)
+ }
+ return out
+}
+
+// HeldFor is everything the mesh keeps that is keyed on one module.
+//
+// Read rather than counted at the moment of removal, because the answer is the whole of what a
+// person needs in order to say yes.
+func (i *Inventory) HeldFor(ctx context.Context, name string) (Holdings, error) {
+ var held Holdings
+ rows, err := i.store.Pool().Query(ctx,
+ `select coalesce(n.name, '') from settings s left join node n on n.id = s.node
+ where s.module = $1 order by n.name nulls first`, name)
+ if err != nil {
+ return Holdings{}, err
+ }
+ for rows.Next() {
+ var node string
+ if err := rows.Scan(&node); err != nil {
+ rows.Close()
+ return Holdings{}, err
+ }
+ if node == "" {
+ held.Mesh = true
+ continue
+ }
+ held.Nodes = append(held.Nodes, node)
+ }
+ rows.Close()
+ if err := rows.Err(); err != nil {
+ return Holdings{}, err
+ }
+
+ for _, read := range []struct {
+ query string
+ into *[]string
+ }{
+ {`select s.name || ' on ' || n.name from module_secret s join node n on n.id = s.node
+ where s.module = $1 order by n.name, s.name`, &held.Secrets},
+ {`select p.wanted::text || ' on ' || n.name from port_assignment p
+ join node n on n.id = p.node where p.module = $1 order by n.name, p.wanted`, &held.Ports},
+ } {
+ rows, err := i.store.Pool().Query(ctx, read.query, name)
+ if err != nil {
+ return Holdings{}, err
+ }
+ for rows.Next() {
+ var one string
+ if err := rows.Scan(&one); err != nil {
+ rows.Close()
+ return Holdings{}, err
+ }
+ *read.into = append(*read.into, one)
+ }
+ rows.Close()
+ if err := rows.Err(); err != nil {
+ return Holdings{}, err
+ }
+ }
+ return held, nil
+}
+
+// ForgetModule removes a module, unless a machine is running it, unless the mesh still holds
+// things for it, and never one the control plane provides.
+//
+// **Refused rather than cascaded.** The settings, own-secrets and port assignments all name the
+// module by a foreign key that cascades, so the row going takes them with it and says nothing.
+// That is an action succeeding into a state its own verify would reject (novox/hq 04-ISSUES/017):
+// the command reports "forgotten", the operator re-registers the module a moment later, and what
+// comes back is a module with none of its configuration and none of its secrets — with nothing
+// anywhere naming the moment they were lost.
func (i *Inventory) ForgetModule(ctx context.Context, name string) error {
+ if err := i.mayForget(ctx, name); err != nil {
+ return err
+ }
+ held, err := i.HeldFor(ctx, name)
+ if err != nil {
+ return err
+ }
+ if held.Any() {
+ return fmt.Errorf("%w:\n%s\n\nAll of it goes when the module does. Run "+
+ "`module forget %s --and-what-it-holds` if that is what you mean",
+ ErrStillHolds, strings.Join(held.Lines(), "\n"), name)
+ }
+ return i.discard(ctx, name)
+}
+
+// DiscardModule removes a module and everything the mesh holds for it, having been told to.
+//
+// The same checks as ForgetModule except the one about what is held: a machine running it still
+// refuses, and a module the control plane provides still refuses, because neither of those is
+// something an operator can consent to on the module's behalf.
+func (i *Inventory) DiscardModule(ctx context.Context, name string) (Holdings, error) {
+ if err := i.mayForget(ctx, name); err != nil {
+ return Holdings{}, err
+ }
+ held, err := i.HeldFor(ctx, name)
+ if err != nil {
+ return Holdings{}, err
+ }
+ if err := i.discard(ctx, name); err != nil {
+ return Holdings{}, err
+ }
+ // Returned so the caller can say what went, rather than "forgotten". A person who has just
+ // destroyed a sealed secret should be able to read which one from the output.
+ return held, nil
+}
+
+// mayForget is the part of forgetting that is not about what is held.
+func (i *Inventory) mayForget(ctx context.Context, name string) error {
provided, err := i.Provided(ctx, name)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return fmt.Errorf("%w: %s", ErrNoSuchModule, name)
+ }
if err != nil {
return err
}
@@ -235,10 +390,17 @@ func (i *Inventory) ForgetModule(ctx context.Context, name string) error {
on = append(on, node)
}
rows.Close()
+ if err := rows.Err(); err != nil {
+ return err
+ }
if len(on) > 0 {
return fmt.Errorf("%w: %s. Unassign it first", ErrStillAssigned, strings.Join(on, ", "))
}
+ return nil
+}
+// discard is the removal itself, once it has been decided.
+func (i *Inventory) discard(ctx context.Context, name string) error {
tag, err := i.store.Pool().Exec(ctx, `delete from module where name = $1`, name)
if err != nil {
return err
diff --git a/internal/inventory/forget_test.go b/internal/inventory/forget_test.go
new file mode 100644
index 0000000..31e3b02
--- /dev/null
+++ b/internal/inventory/forget_test.go
@@ -0,0 +1,226 @@
+package inventory
+
+import (
+ "errors"
+ "strings"
+ "testing"
+
+ "github.com/novox/mesh-control/internal/catalogue"
+)
+
+// What removing a module takes with it, and what re-registering one does not.
+//
+// Both were reported as one fault — "operator settings do not persist, because re-registering a
+// module cascade-deletes them". Only half of it was true, and it was the other half.
+
+// **Registering a module again does not touch what the mesh holds for it.** The upsert is on the
+// name, so a manifest changing — a new version, a new requirement, a new resource — leaves the
+// settings, the secrets and the ports exactly where they were.
+//
+// This is here because it was believed not to be. A wrong belief about which command destroys data
+// is expensive in both directions: it sends people looking for a fault that is not there, and it
+// leaves the command that really does destroy it unexamined.
+func TestRegisteringAModuleAgainKeepsWhatTheMeshHoldsForIt(t *testing.T) {
+ inv := fresh(t)
+ ctx := t.Context()
+ node, err := inv.AddNode(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, _ := aSealingKey(t)
+ if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
+ t.Fatal(err)
+ }
+ m := catalogue.Manifest{Module: "step-ca", Version: "1",
+ Provides: catalogue.Offers("acme-ca")}
+ if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.SetSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.SetSettings(ctx, "anchor", "step-ca", map[string]any{"port": 9000}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.AcceptSecretForModule(ctx, "anchor", "step-ca", "password", "sealed"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := inv.PortFor(ctx, "anchor", "step-ca", 9000, false); err != nil {
+ t.Fatal(err)
+ }
+
+ // Re-registered at a new version, which is exactly what somebody bumping one does.
+ m.Version = "2"
+ if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
+ t.Fatal(err)
+ }
+
+ layers, err := inv.SettingsFor(ctx, "anchor", "step-ca")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(layers) != 2 {
+ t.Fatalf("re-registering lost a settings layer: %+v", layers)
+ }
+ held, err := inv.HeldFor(ctx, "step-ca")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !held.Mesh || len(held.Nodes) != 1 || len(held.Secrets) != 1 || len(held.Ports) != 1 {
+ t.Fatalf("re-registering lost something the mesh held: %+v", held)
+ }
+
+ // And the new manifest is what resolution sees, which is the point of re-registering at all.
+ shelf, err := inv.Catalogue(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if shelf["step-ca"].Version != "2" {
+ t.Fatalf("the new manifest did not replace the old: %+v", shelf["step-ca"])
+ }
+ if len(shelf["step-ca"].Provides) != 1 || shelf["step-ca"].Provides[0].Name != "acme-ca" {
+ // A version bump was reported as un-providing a provision. It does not.
+ t.Fatalf("a version bump changed what the module provides: %+v", shelf["step-ca"].Provides)
+ }
+}
+
+// **Forgetting a module refuses while the mesh still holds things for it, and says what they are.**
+//
+// The settings, the module's own secrets and the ports the mesh chose all name the module by a
+// foreign key that cascades. So the removal took them, silently, and reported "forgotten" — an
+// action succeeding into a state its own verify would reject (novox/hq 04-ISSUES/017). A sealed
+// secret is not recoverable afterwards: the mesh discarded the plaintext when it made it.
+func TestForgettingAModuleRefusesRatherThanDiscardingWhatTheMeshHolds(t *testing.T) {
+ inv := fresh(t)
+ ctx := t.Context()
+ node, err := inv.AddNode(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, _ := aSealingKey(t)
+ if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.RegisterModule(ctx, manifest("step-ca", nil, nil), Source{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.SetSettings(ctx, "anchor", "step-ca", map[string]any{"port": 9000}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.AcceptSecretForModule(ctx, "anchor", "step-ca", "password", "sealed"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := inv.PortFor(ctx, "anchor", "step-ca", 9000, false); err != nil {
+ t.Fatal(err)
+ }
+
+ err = inv.ForgetModule(ctx, "step-ca")
+ if !errors.Is(err, ErrStillHolds) {
+ t.Fatalf("forgetting discarded what the mesh held, or refused for another reason: %v", err)
+ }
+ // It names them one at a time. "3 things" says there is something to lose and not whether it
+ // can be afforded; the sealed secret is the one that cannot be made again, and it is named.
+ for _, want := range []string{"settings, on anchor", "password on anchor",
+ "the mesh cannot make it again", "the port 9000 on anchor", "--and-what-it-holds"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("the refusal does not say %q:\n%s", want, err)
+ }
+ }
+ // And nothing went. A refusal that half-happened would be worse than the cascade.
+ shelf, err := inv.Catalogue(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, still := shelf["step-ca"]; !still {
+ t.Fatal("the module was removed by a command that refused")
+ }
+ layers, err := inv.SettingsFor(ctx, "anchor", "step-ca")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(layers) != 1 {
+ t.Fatalf("a refusal took the settings anyway: %+v", layers)
+ }
+}
+
+// Having been told, it goes — and what went is reported, because this is the only record that any
+// of it existed.
+func TestDiscardingAModuleSaysWhatWentWithIt(t *testing.T) {
+ inv := fresh(t)
+ ctx := t.Context()
+ node, err := inv.AddNode(ctx, "anchor")
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, _ := aSealingKey(t)
+ if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.RegisterModule(ctx, manifest("step-ca", nil, nil), Source{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.SetSettings(ctx, "", "step-ca", map[string]any{"issuer": "the mesh"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.AcceptSecretForModule(ctx, "anchor", "step-ca", "password", "sealed"); err != nil {
+ t.Fatal(err)
+ }
+
+ held, err := inv.DiscardModule(ctx, "step-ca")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !held.Mesh || len(held.Secrets) != 1 {
+ t.Fatalf("what went was not reported: %+v", held)
+ }
+ shelf, err := inv.Catalogue(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, still := shelf["step-ca"]; still {
+ t.Fatal("the module is still there")
+ }
+}
+
+// A module the mesh holds nothing for is forgotten without ceremony. The refusal is about loss,
+// not about the command.
+func TestForgettingAModuleTheMeshHoldsNothingForJustWorks(t *testing.T) {
+ inv := fresh(t)
+ ctx := t.Context()
+ if _, err := inv.AddNode(ctx, "anchor"); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.RegisterModule(ctx, manifest("plain", nil, nil), Source{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.ForgetModule(ctx, "plain"); err != nil {
+ t.Fatalf("a module holding nothing was refused: %v", err)
+ }
+}
+
+// A module still on a machine refuses first, whatever it holds: that is not a loss an operator can
+// consent to on the machine's behalf, and unassign is what "I do not want this" means.
+func TestAModuleStillAssignedRefusesBeforeAnythingAboutWhatItHolds(t *testing.T) {
+ inv := fresh(t)
+ ctx := t.Context()
+ if _, err := inv.AddNode(ctx, "anchor"); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.RegisterModule(ctx, manifest("step-ca", nil, nil), Source{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.SetSettings(ctx, "anchor", "step-ca", map[string]any{"a": 1}); err != nil {
+ t.Fatal(err)
+ }
+ if err := inv.Assign(ctx, "anchor", "step-ca"); err != nil {
+ t.Fatal(err)
+ }
+ for _, forget := range []func() error{
+ func() error { return inv.ForgetModule(ctx, "step-ca") },
+ func() error { _, err := inv.DiscardModule(ctx, "step-ca"); return err },
+ } {
+ if err := forget(); !errors.Is(err, ErrStillAssigned) {
+ t.Fatalf("an assigned module was not refused for being assigned: %v", err)
+ }
+ }
+}