Files
mesh-controller/cmd/mesh-controller/nodes.go
T
jschoubben 54812306be A machine says which networks it routes, and its filter forwards them
The derived filter denies forwarding by default and then allows the container
runtime's two default pools, named in this code with a comment saying a machine
configured otherwise needs to say so -- and no way to say it. So the filter was
right on a machine using the defaults and silently wrong on any other.

Measured today: flipping a workstation to the derived filter cut egress for five
of its container networks and for every network its test beds create, because
those come from ranges the defaults do not cover. Nothing reported a fault; the
guests just could not reach anything, while the machine reported it had applied
what it was told.

A node-level fact beside the public domain, because the machine routes them and
the module that loads the filter may be replaced. Added to the defaults, never
replacing them. Their guests also keep address and name service, without which a
network does not work at all, and the converge preview now says what a machine
routes instead of leaving it to a sentence about what it cannot preview.
2026-09-28 21:29:10 +02:00

539 lines
19 KiB
Go

package main
import (
"context"
"errors"
"flag"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/token"
)
// what a machine is, and what it is allowed to be told.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
func nodeCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage +
", or " + networksUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
switch args[0] {
case "show":
if len(args) != 2 {
return errors.New("node show <name>")
}
return showNode(ctx, inv, args[1])
case "add":
return addNode(ctx, inv, args[1:])
case "list":
nodes, err := inv.Nodes(ctx)
if err != nil {
return err
}
if len(nodes) == 0 {
// Said rather than printed as nothing: an empty list and a failed read must never
// look the same, and this command answering "none" is only honest because getting
// here means the store answered.
fmt.Println("this mesh has no node records yet")
return nil
}
for _, n := range nodes {
fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID)
}
return nil
case "public-domain":
// The domain this node composes its routed names under (novox/hq ADR 0066).
//
// **The form with no argument reports; clearing is asked for by name.** It used to clear —
// so `node public-domain anchor`, which reads like a question and is what anybody types to
// find out what the answer is, silently took every routed name the node had. A read-shaped
// invocation must never be a destructive write: there is no output that makes up for it,
// because the damage is already done by the time it prints.
return publicDomain(ctx, inv, args[1:])
case "networks":
// The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived
// filter must keep forwarding, beyond the container runtime's own default pools which it
// allows without being told. Reports with no argument, for the same reason the domain does.
return nodeNetworks(ctx, inv, args[1:])
case "account":
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
// an optional second argument is the home when it is not /home/<account>.
return nodeAccount(ctx, inv, args[1:])
default:
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
}
}
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
set := flag.NewFlagSet("node add", flag.ContinueOnError)
adopted := set.Bool("adopted", false,
"the machine is in use: keep what is found on it until each module is taken")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("node add <name> [--adopted]")
}
node, err := inv.AddNodeAs(ctx, positionals[0], *adopted)
if err != nil {
return err
}
fmt.Printf("added %s (%s)", node.Name, node.ID)
if node.Adopted {
fmt.Print(", adopted")
}
fmt.Println()
return nil
}
// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted
// wherever the mesh reports a node's state.
func modeOf(n inventory.Node) string {
if n.Adopted {
return "adopted"
}
return "converged"
}
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
// argument, like public-domain: `node account novox` answers, it does not change anything.
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
if len(positionals) == 0 || len(positionals) > 3 {
return errors.New("node account <name> — what it is now; " +
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
}
node := positionals[0]
if len(positionals) == 1 {
who, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
if who.Account == "" {
fmt.Printf("%s has no operator account known\n", node)
fmt.Printf(" `node account %s <account>` sets it\n", node)
return nil
}
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
return nil
}
home := ""
if len(positionals) == 3 {
home = positionals[2]
}
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
return err
}
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
return nil
}
const networksUsage = "node networks <name> — what it routes now; " +
"<name> <cidr>... to set them; <name> --clear to route only the container runtime's own"
// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts.
//
// The same three forms as the domain above, and the read-shaped one reports rather than clearing,
// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it
// by asking a question is not a mistake anybody can see afterwards.
func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error {
set := flag.NewFlagSet("node networks", flag.ContinueOnError)
clear := set.Bool("clear", false,
"route only the container runtime's own default pools, as a machine that has said nothing does")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) == 0 {
return errors.New(networksUsage)
}
node := positionals[0]
switch {
case *clear && len(positionals) > 1:
return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+
"things and the mesh will not choose between them", node, strings.Join(positionals[1:], " "))
case *clear:
if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil {
return err
}
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
fmt.Printf(" run `push %s` to send its filter\n", node)
return nil
case len(positionals) > 1:
if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil {
return err
}
fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", "))
fmt.Printf(" its filter forwards them, and their guests keep address and name service\n")
fmt.Printf(" run `push %s` to send it\n", node)
return nil
default:
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
networks, err := inv.RoutedNetworksOf(ctx, node)
if err != nil {
return err
}
if len(networks) == 0 {
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
fmt.Printf(" `node networks %s <cidr>...` if its guests live elsewhere\n", node)
return nil
}
fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", "))
return nil
}
}
const publicDomainUsage = "node public-domain <name> — what it is now; " +
"<name> <domain> to set it; <name> --clear to take it away"
// publicDomain reads, sets or clears the domain a node composes its routed names under.
//
// Three forms, and the destructive one is the only one that has to be asked for. Clearing is a
// real thing to want — a machine that stops facing the outside composes no names, and
// lab-versus-production is this one setting (novox/hq ADR 0066) — so it keeps a way to say it.
// What it does not keep is being the thing that happens when nothing was said at all.
func publicDomain(ctx context.Context, inv *inventory.Inventory, args []string) error {
set := flag.NewFlagSet("node public-domain", flag.ContinueOnError)
clear := set.Bool("clear", false, "take the domain away; it composes no routed names after")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) == 0 || len(positionals) > 2 {
return errors.New(publicDomainUsage)
}
node := positionals[0]
switch {
case *clear && len(positionals) == 2:
// Both, which cannot be meant. Refused rather than one of them silently winning.
return fmt.Errorf("give %s a domain or --clear, not both: %q and --clear say opposite "+
"things and the mesh will not choose between them", node, positionals[1])
case *clear:
if err := inv.SetPublicDomain(ctx, node, ""); err != nil {
return err
}
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
fmt.Printf(" run `push %s` to take them off it\n", node)
return nil
case len(positionals) == 2:
if err := inv.SetPublicDomain(ctx, node, positionals[1]); err != nil {
return err
}
fmt.Printf("%s composes its routed names under %s\n", node, positionals[1])
fmt.Printf(" run `push %s` to send it\n", node)
return nil
default:
// Asked, so answered. NodeByName first, so a name the mesh has never heard of is a refusal
// rather than "it has no public domain", which is true of that name and says nothing.
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
domain, err := inv.PublicDomainOf(ctx, node)
if err != nil {
return err
}
if domain == "" {
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
fmt.Printf(" `node public-domain %s <domain>` gives it one\n", node)
return nil
}
fmt.Printf("%s composes its routed names under %s\n", node, domain)
return nil
}
}
func tokenCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "issue" {
return errors.New("token issue --node <name>, or token issue --new <name>")
}
set := flag.NewFlagSet("token issue", flag.ContinueOnError)
existing := set.String("node", "", "issue for a node record that already exists")
fresh := set.String("new", "", "create the node record, then issue for it")
validFor := set.Duration("for", time.Hour, "how long the token may be used")
adopted := set.Bool("adopted", false,
"the machine joining is in use: it is adopted, and keeps what is found on it")
if err := set.Parse(args[1:]); err != nil {
return err
}
// Exactly one, because the difference is what the token binds to. A command that guessed
// would sometimes create a second record for a machine that already has one.
if (*existing == "") == (*fresh == "") {
return errors.New("give exactly one of --node <name> or --new <name>: the first is a " +
"machine the mesh already has a record for, the second is one it has never seen")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor)
if err != nil {
return err
}
// Assembled from two contexts by the process that holds both grants. Neither reads the
// other's store (novox/hq ADR 0008) — each is asked for its own part.
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
key, err := ident.Establish(ctx)
if err != nil {
return err
}
// The account is created before the token is handed over, which is what removes the
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
// exist before it does. The one-time secret IS the password, so a node's first connection is
// already authenticated and enrolment is what happens over it.
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
Adopted: issued.Node.Adopted}
// Absent is a state, not a failure: a control plane can hold records and a key before it has
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
known, err := broker.FromEnvironment()
switch {
case err == nil:
made.Broker, made.Fingerprint = known.Address, known.Fingerprint
case errors.Is(err, broker.ErrNotConfigured):
default:
return err
}
encoded, err := made.Encode()
if err != nil {
return err
}
joins := ""
if made.Adopted {
joins = ", joining adopted"
}
fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n",
issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded)
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
if missing := made.Missing(); len(missing) > 0 {
fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n")
for _, m := range missing {
fmt.Printf(" - %s\n", m)
}
fmt.Printf("\nSet %s and %s once the broker is raised.\n",
broker.AddressVar, broker.CertificateVar)
}
return nil
}
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
// when the operator says so, and the one-time secret for it. The node in what it returns carries
// its mode, which is what the token says.
func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool,
validFor time.Duration) (inventory.Issued, error) {
name := existing
if fresh != "" {
node, err := inv.AddNodeAs(ctx, fresh, adopted)
if err != nil {
return inventory.Issued{}, err
}
name = node.Name
}
// Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not
// converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a
// node already converged is refused rather than done quietly: returning a node to adopted is
// its own act too, which unloads the mesh's filter and enables the found firewall again.
if adopted && fresh == "" {
node, err := inv.NodeByName(ctx, name)
if err != nil {
return inventory.Issued{}, err
}
if !node.Adopted {
return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+
"that: run `adopt %s` to return it to adopted, then issue the token without "+
"--adopted", name, name)
}
}
return inv.IssueToken(ctx, name, validFor)
}
func identityCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("identity show")
}
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
// Establish rather than read: a control plane asked for its identity before it has one should
// get one, not an error. Generating it is idempotent, so this is safe to run at any time.
key, err := ident.Establish(ctx)
if err != nil {
return err
}
fmt.Printf("signing key %s\n", key.ID)
fmt.Printf("fingerprint %s\n", key.Fingerprint())
fmt.Printf("created %s\n", key.Created.Format(time.RFC3339))
fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" +
"declaration because it carries a signature this key made (novox/hq ADR 0004).\n")
return nil
}
func brokerCommand(args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show")
}
known, err := broker.FromEnvironment()
if errors.Is(err, broker.ErrNotConfigured) {
fmt.Printf("no broker configured. Set %s and %s.\n\n"+
"Until then tokens carry the signing key and the one-time secret, and say what they\n"+
"are missing. They cannot be used to join.\n",
broker.AddressVar, broker.CertificateVar)
return nil
}
if err != nil {
return err
}
fmt.Printf("address %s\n", known.Address)
fmt.Printf("fingerprint %s\n", known.Fingerprint)
fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" +
"node checks it before sending anything (novox/hq ADR 0004).\n")
return nil
}
// heardFrom says when a node was last heard from, in a form somebody can act on.
//
// "never" and "an hour ago" are different answers and are kept different. A node that has never
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
// picture of the mesh.
func heardFrom(n inventory.Node) string {
silent, ever := n.Silent()
switch {
case !ever:
return "never spoken"
case silent > SilentFor:
return "out of touch " + roughly(silent)
default:
return "here"
}
}
// roughly is a duration a person reads rather than parses.
func roughly(d time.Duration) string {
switch {
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%dh", int(d.Hours()))
default:
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
}
// showNode says what one machine reported about itself, in its own words.
//
// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what
// a machine can do is the one it gave — and its detail is half of that account. The mesh was
// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with
// nowhere to go: a person told a machine lacks something wants to know what the detector saw.
//
// It is also where "what should it be configured as" is read. The same line that gates an
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
node, err := inv.NodeByName(ctx, name)
if err != nil {
return err
}
fmt.Printf("%s\n", node.Name)
fmt.Printf(" last heard from %s\n", heardFrom(node))
if err := showMode(ctx, inv, node); err != nil {
return err
}
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
// every internal node would be noise.
domain, err := inv.PublicDomainOf(ctx, name)
if err != nil {
return err
}
if domain != "" {
fmt.Printf(" public domain %s\n", domain)
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
}
if held == nil {
// Never reported is not the same as reported nothing, and the remedy differs: one is a
// machine that has not run the host yet, the other is a machine that ran it and can do
// nothing.
fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" +
" capability is refused here — run the host on it\n")
return nil
}
if len(held) == 0 {
fmt.Printf("\n it reported no capabilities at all\n")
return nil
}
fmt.Printf("\n what it can do, as it reported:\n")
for _, c := range held {
mark := "no "
if c.Present {
mark = "yes"
}
fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail)
}
assigned, err := inv.Assigned(ctx, name)
if err != nil {
return err
}
if len(assigned) > 0 {
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
}
return nil
}