jschoubben
7d033ad9f6
One store, and it is the registry the bootstrap already pulls from. An OCI registry is a content-addressed blob store that also understands images: PUT a blob and it is retrievable at /v2/<name>/blobs/sha256:… for ever, by digest. An archive is a content-addressed blob. A second store beside it was considered and is the right answer for objects that are mutable, need per-reader access, or are not build output — somebody's uploads, a backup, a thing with a lifecycle. None of that describes a digest-pinned archive, and running a second service to hold one kind of immutable blob is two things to run, two to back up, and two ways for an artifact to be missing. Overturnable by reading: the manifest carries a URL and a digest, and neither says what served it. `build <repository>` clones, reads module.json, builds what it declares, publishes, and records the manifest with the commit it came from. It is a command rather than something the control plane does on its own, because building runs things on a machine and what the control plane may send a machine is bounded by the declaration language. This is the shape the builder module takes when it is given work over the broker. Proven end to end on a real repository and a real registry: a shell module with a package, a user and a dotfile archive built, published, fetched back at the digest it declared, rebuilt to the same digest, and its manifest accepted by the host's own parser — including `user` and `archive`, which did not exist this morning. A tag is never accepted as a pin, and a blob already stored is not sent again — it is named by its content, so re-uploading asks the registry to store what it already has under the name it already has.
1695 lines
55 KiB
Go
1695 lines
55 KiB
Go
// Command mesh-control is the control plane: everything that needs to know about more than one
|
|
// node (novox/hq ADR 0006).
|
|
//
|
|
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
|
|
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
|
|
// bootstrap in novox/hq 07-the-substrate and the step the first node cannot get past without.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-control/internal/broker"
|
|
"github.com/novox/mesh-control/internal/builder"
|
|
"github.com/novox/mesh-control/internal/catalogue"
|
|
"github.com/novox/mesh-control/internal/identity"
|
|
"github.com/novox/mesh-control/internal/inventory"
|
|
"github.com/novox/mesh-control/internal/link"
|
|
"github.com/novox/mesh-control/internal/overlay"
|
|
"github.com/novox/mesh-control/internal/store"
|
|
"github.com/novox/mesh-control/internal/token"
|
|
)
|
|
|
|
// version is stamped at link time. Unset in a development build, and it says so rather than
|
|
// claiming a number.
|
|
var version = "development build"
|
|
|
|
// held is a context this process was granted, and the schema it carries.
|
|
//
|
|
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
|
|
// yet, not because they are optional.
|
|
var held = []struct {
|
|
name string
|
|
migrations func() ([]store.Migration, error)
|
|
}{
|
|
{inventory.Name, inventory.Migrations},
|
|
{identity.Name, identity.Migrations},
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-control: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
args := os.Args[1:]
|
|
if len(args) == 0 {
|
|
usage()
|
|
return fmt.Errorf("no command given")
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
switch args[0] {
|
|
case "build":
|
|
return buildCommand(ctx, args[1:])
|
|
case "pin":
|
|
return pinCommand(ctx, args[1:], true)
|
|
case "unpin":
|
|
return pinCommand(ctx, args[1:], false)
|
|
case "migrate":
|
|
return migrate(ctx)
|
|
case "node":
|
|
return nodeCommand(ctx, args[1:])
|
|
case "token":
|
|
return tokenCommand(ctx, args[1:])
|
|
case "identity":
|
|
return identityCommand(ctx, args[1:])
|
|
case "broker":
|
|
return brokerCommand(args[1:])
|
|
case "serve":
|
|
return serve(ctx)
|
|
case "declare":
|
|
return declare(ctx, args[1:])
|
|
case "overlay":
|
|
return overlayCommand(ctx, args[1:])
|
|
case "module":
|
|
return moduleCommand(ctx, args[1:])
|
|
case "assign", "unassign":
|
|
return assignCommand(ctx, args[0], args[1:])
|
|
case "settings":
|
|
return settingsCommand(ctx, args[1:])
|
|
case "plan":
|
|
return planCommand(ctx, args[1:])
|
|
case "push":
|
|
return pushCommand(ctx, args[1:])
|
|
case "status":
|
|
return statusCommand(ctx)
|
|
case "version":
|
|
fmt.Println(version)
|
|
return nil
|
|
case "help", "-h", "--help":
|
|
usage()
|
|
return nil
|
|
default:
|
|
usage()
|
|
return fmt.Errorf("%q is not a command", args[0])
|
|
}
|
|
}
|
|
|
|
func usage() {
|
|
fmt.Fprint(os.Stderr, `mesh-control — the control plane
|
|
|
|
migrate bring each context's schema up to date
|
|
node add <name> create a node record
|
|
node list the nodes this mesh knows about
|
|
token issue --node <name> a one-time right to join, for an existing record
|
|
token issue --new <name> create the record and issue for it
|
|
identity show this control plane's signing key
|
|
broker show where the broker is, and what to expect there
|
|
serve consume what nodes say, and answer
|
|
declare <node> <file> send a node a signed declaration
|
|
overlay place <node> [flags] say where a node is and how it is reached
|
|
overlay show the private network, as the mesh computes it
|
|
module add <file> register a module from its manifest
|
|
module list what modules this mesh knows about
|
|
module moved <name> <commit> the source has a newer commit than the mesh built
|
|
module forget <name> remove one, unless a node is running it
|
|
status what the mesh is behind on, and which nodes
|
|
assign <node> <module> put a module on a node
|
|
unassign <node> <module> take it off
|
|
settings set <module> <file> what a module's config should say, for the whole mesh
|
|
settings set <module> <file> --node <n> ...or for one machine
|
|
settings clear <module> [--node <n>] take a layer away
|
|
build <repository> [--ref R] build a module from its source and record it
|
|
pin <node> <provision> <from> which node this one gets a provision from
|
|
unpin <node> <provision> put that question back
|
|
plan <node> [--files|--json] what that node would run, and why
|
|
push [<node>] send a node everything it should be
|
|
version what this binary is
|
|
|
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
|
`+store.Variable("<context>")+`. This process holds:
|
|
|
|
`)
|
|
for _, c := range held {
|
|
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
|
c.name, store.Database(c.name), store.Variable(c.name))
|
|
}
|
|
fmt.Fprintln(os.Stderr)
|
|
}
|
|
|
|
// migrate brings every held context's schema up to date.
|
|
//
|
|
// Reported per context and per migration, because this runs during a bootstrap on a machine with
|
|
// nothing else on it — the output is the only account of what happened, and "migrated" is not one.
|
|
func migrate(ctx context.Context) error {
|
|
for _, c := range held {
|
|
migrations, err := c.migrations()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s, err := store.Open(ctx, c.name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer s.Close()
|
|
|
|
// The bootstrap raises PostgreSQL moments before this runs, and a container that is
|
|
// running is not a database that will answer — a distinction this project has already
|
|
// paid for once, when a crash-looping database reported itself as up between restarts.
|
|
if err := s.Ready(ctx, 60*time.Second); err != nil {
|
|
return err
|
|
}
|
|
|
|
done, err := s.Migrate(ctx, migrations)
|
|
for _, m := range done {
|
|
fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(done) == 0 {
|
|
applied, err := s.AppliedMigrations(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied))
|
|
}
|
|
}
|
|
|
|
// The modules the control plane ships with itself. Recorded here rather than by hand, because
|
|
// a mesh whose own private network is missing from the catalogue would have nothing to assign
|
|
// and no way to say why.
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
for _, m := range provided {
|
|
if err := inv.Provide(ctx, m); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("provided %s\n", m.Module)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// provided is what comes with the control plane rather than from a repository.
|
|
//
|
|
// WireGuard, the names, and the domain module over both. The first two are here because the code
|
|
// that works out their files is here:
|
|
// a peer list is derived from every machine at once, so it cannot be written in a manifest, and
|
|
// whatever computes it has to live wherever the whole picture is.
|
|
//
|
|
// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent
|
|
// from a machine nobody gave it to.
|
|
func providedModules() []catalogue.Manifest {
|
|
var out []catalogue.Manifest
|
|
for _, raw := range []map[string]any{
|
|
overlay.Manifest(), overlay.NamesManifest(), overlay.DomainManifest(),
|
|
} {
|
|
var m catalogue.Manifest
|
|
b, _ := json.Marshal(raw)
|
|
_ = json.Unmarshal(b, &m)
|
|
out = append(out, m)
|
|
}
|
|
return out
|
|
}
|
|
|
|
var provided = providedModules()
|
|
|
|
// openInventory connects and waits, the way every command that touches it needs to.
|
|
func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
|
inv, err := inventory.Open(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := inv.Ready(ctx, 30*time.Second); err != nil {
|
|
inv.Close()
|
|
return nil, err
|
|
}
|
|
return inv, nil
|
|
}
|
|
|
|
func nodeCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 {
|
|
return errors.New("node add <name>, or node list")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
switch args[0] {
|
|
case "add":
|
|
if len(args) != 2 {
|
|
return errors.New("node add <name>")
|
|
}
|
|
node, err := inv.AddNode(ctx, args[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("added %s (%s)\n", node.Name, node.ID)
|
|
return nil
|
|
|
|
case "list":
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(nodes) == 0 {
|
|
// Said rather than printed as nothing: an empty list and a failed read must never
|
|
// look the same, and this command answering "none" is only honest because getting
|
|
// here means the store answered.
|
|
fmt.Println("this mesh has no node records yet")
|
|
return nil
|
|
}
|
|
for _, n := range nodes {
|
|
fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID)
|
|
}
|
|
return nil
|
|
|
|
default:
|
|
return fmt.Errorf("node has no %q; it has add and list", args[0])
|
|
}
|
|
}
|
|
|
|
func tokenCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 || args[0] != "issue" {
|
|
return errors.New("token issue --node <name>, or token issue --new <name>")
|
|
}
|
|
|
|
set := flag.NewFlagSet("token issue", flag.ContinueOnError)
|
|
existing := set.String("node", "", "issue for a node record that already exists")
|
|
fresh := set.String("new", "", "create the node record, then issue for it")
|
|
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
|
if err := set.Parse(args[1:]); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Exactly one, because the difference is what the token binds to. A command that guessed
|
|
// would sometimes create a second record for a machine that already has one.
|
|
if (*existing == "") == (*fresh == "") {
|
|
return errors.New("give exactly one of --node <name> or --new <name>: the first is a " +
|
|
"machine the mesh already has a record for, the second is one it has never seen")
|
|
}
|
|
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
name := *existing
|
|
if *fresh != "" {
|
|
node, err := inv.AddNode(ctx, *fresh)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
name = node.Name
|
|
}
|
|
|
|
issued, err := inv.IssueToken(ctx, name, *validFor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Assembled from two contexts by the process that holds both grants. Neither reads the
|
|
// other's store (novox/hq ADR 0008) — each is asked for its own part.
|
|
ident, err := openIdentity(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ident.Close()
|
|
key, err := ident.Establish(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The account is created before the token is handed over, which is what removes the
|
|
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
|
|
// exist before it does. The one-time secret IS the password, so a node's first connection is
|
|
// already authenticated and enrolment is what happens over it.
|
|
if management, err := broker.ManagementFromEnvironment(); err == nil {
|
|
if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n",
|
|
issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange)
|
|
} else if !errors.Is(err, broker.ErrNotConfigured) {
|
|
return err
|
|
}
|
|
|
|
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret}
|
|
|
|
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
|
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
|
known, err := broker.FromEnvironment()
|
|
switch {
|
|
case err == nil:
|
|
made.Broker, made.Fingerprint = known.Address, known.Fingerprint
|
|
case errors.Is(err, broker.ErrNotConfigured):
|
|
default:
|
|
return err
|
|
}
|
|
encoded, err := made.Encode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n",
|
|
issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded)
|
|
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
|
|
|
if missing := made.Missing(); len(missing) > 0 {
|
|
fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n")
|
|
for _, m := range missing {
|
|
fmt.Printf(" - %s\n", m)
|
|
}
|
|
fmt.Printf("\nSet %s and %s once the broker is raised.\n",
|
|
broker.AddressVar, broker.CertificateVar)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func openIdentity(ctx context.Context) (*identity.Identity, error) {
|
|
ident, err := identity.Open(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := ident.Ready(ctx, 30*time.Second); err != nil {
|
|
ident.Close()
|
|
return nil, err
|
|
}
|
|
return ident, nil
|
|
}
|
|
|
|
func identityCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 || args[0] != "show" {
|
|
return errors.New("identity show")
|
|
}
|
|
ident, err := openIdentity(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ident.Close()
|
|
|
|
// Establish rather than read: a control plane asked for its identity before it has one should
|
|
// get one, not an error. Generating it is idempotent, so this is safe to run at any time.
|
|
key, err := ident.Establish(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("signing key %s\n", key.ID)
|
|
fmt.Printf("fingerprint %s\n", key.Fingerprint())
|
|
fmt.Printf("created %s\n", key.Created.Format(time.RFC3339))
|
|
fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" +
|
|
"declaration because it carries a signature this key made (novox/hq ADR 0004).\n")
|
|
return nil
|
|
}
|
|
|
|
func brokerCommand(args []string) error {
|
|
if len(args) == 0 || args[0] != "show" {
|
|
return errors.New("broker show")
|
|
}
|
|
known, err := broker.FromEnvironment()
|
|
if errors.Is(err, broker.ErrNotConfigured) {
|
|
fmt.Printf("no broker configured. Set %s and %s.\n\n"+
|
|
"Until then tokens carry the signing key and the one-time secret, and say what they\n"+
|
|
"are missing. They cannot be used to join.\n",
|
|
broker.AddressVar, broker.CertificateVar)
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("address %s\n", known.Address)
|
|
fmt.Printf("fingerprint %s\n", known.Fingerprint)
|
|
fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" +
|
|
"node checks it before sending anything (novox/hq ADR 0004).\n")
|
|
return nil
|
|
}
|
|
|
|
// serve is the control plane running: one connection to the broker, one queue, one consumer.
|
|
func serve(ctx context.Context) error {
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
ident, err := openIdentity(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ident.Close()
|
|
|
|
// Established at start rather than on first use. A control plane that cannot sign is one
|
|
// whose declarations every node correctly refuses, and that should be a startup failure
|
|
// rather than something discovered at the first declaration.
|
|
key, err := ident.Establish(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("signing as %s\n", key.Fingerprint()[:16])
|
|
|
|
management, err := broker.ManagementFromEnvironment()
|
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
|
return err
|
|
}
|
|
|
|
// Where the broker is and what to expect there, so a node can be told how to come back
|
|
// without a person and a new token.
|
|
known, err := broker.FromEnvironment()
|
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
|
return err
|
|
}
|
|
if errors.Is(err, broker.ErrNotConfigured) {
|
|
fmt.Printf("no broker address configured, so enrolled nodes will not be told how to "+
|
|
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
|
|
}
|
|
|
|
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known}
|
|
server, err := link.Connect(work, work)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer server.Close()
|
|
|
|
return server.Serve(ctx)
|
|
}
|
|
|
|
// declare sends one node a declaration, signed.
|
|
//
|
|
// Signed here rather than trusted from the broker: a node connects to the broker and takes
|
|
// instruction from the control plane behind it, and those are two identities. If a node believed
|
|
// whatever arrived on its queue, a compromised broker could forge declarations — and since the
|
|
// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004).
|
|
func declare(ctx context.Context, args []string) error {
|
|
if len(args) != 2 {
|
|
return errors.New("declare <node> <declaration.json>")
|
|
}
|
|
node, path := args[0], args[1]
|
|
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ident, err := openIdentity(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ident.Close()
|
|
|
|
// The node has to exist before it can be told anything. Publishing to a queue nobody consumes
|
|
// would sit there looking like success.
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
|
return err
|
|
}
|
|
|
|
server, err := link.Connect(nil, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer server.Close()
|
|
|
|
if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
|
return nil
|
|
}
|
|
|
|
// OverlayCIDRVar is the range the mesh allocates node addresses from.
|
|
const OverlayCIDRVar = "MESH_OVERLAY_CIDR"
|
|
|
|
func overlayCIDR() string {
|
|
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
|
return v
|
|
}
|
|
return "10.42.0.0/16"
|
|
}
|
|
|
|
func overlayCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 {
|
|
return errors.New("overlay place <node> [flags], or overlay show")
|
|
}
|
|
// Answered before anything is opened. A message about which command to use should not need a
|
|
// database to say so, and needing one turns a redirect into a connection error.
|
|
if args[0] == "push" {
|
|
return errors.New("`overlay push` is now `push`, which sends a node its network AND " +
|
|
"what its assignments resolve to — the two are computed from one picture of the " +
|
|
"mesh, and sending them separately would let them disagree")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
switch args[0] {
|
|
case "place":
|
|
return overlayPlace(ctx, inv, args[1:])
|
|
case "show":
|
|
return overlayShow(ctx, inv)
|
|
|
|
default:
|
|
return fmt.Errorf("overlay has no %q; it has place and show", args[0])
|
|
}
|
|
}
|
|
|
|
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
|
if len(args) == 0 {
|
|
return errors.New("overlay place <node> [--endpoint host:port] [--site name] [--hub]")
|
|
}
|
|
node := args[0]
|
|
|
|
set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
|
|
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
|
|
site := set.String("site", "", "where this machine physically is, or empty if it roams")
|
|
hub := set.Bool("hub", false, "this node is the hub every other routes through")
|
|
if err := set.Parse(args[1:]); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
|
// election by address prefix fails silently (novox/hq ADR 0007).
|
|
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
|
return err
|
|
}
|
|
found, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Printf("%s is at %s on the overlay\n", node, address)
|
|
switch {
|
|
case *hub:
|
|
fmt.Println(" the hub — every node not sharing a site routes through it")
|
|
case *endpoint == "":
|
|
fmt.Println(" not dialable — it opens every path itself")
|
|
}
|
|
if *site != "" {
|
|
fmt.Printf(" at %s, so it peers directly with anything else there\n", *site)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// network builds the private network over the machines that resolved the module for it.
|
|
//
|
|
// Not over every node the mesh knows. **A machine is on the private network because it was given
|
|
// the module**, and one that was not is absent from every peer list and from the names — which is
|
|
// the only thing "not on the network" can mean. Until this, having an address was enough, and
|
|
// there was no way to keep a machine off.
|
|
//
|
|
// Every node at once, which is the whole reason this is the control plane's work: a peer list is
|
|
// derived from all the others, so no node could compute its own.
|
|
func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
|
refused map[string]string) (*overlay.Generator, error) {
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodes := make([]overlay.Node, 0, len(places))
|
|
for _, p := range places {
|
|
if !on[p.Name] {
|
|
continue
|
|
}
|
|
nodes = append(nodes, overlay.Node{
|
|
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
|
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
|
})
|
|
}
|
|
if len(nodes) == 0 {
|
|
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
|
// answers rather than refusing -- Compute would refuse for want of a hub, and reporting
|
|
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
|
return overlay.Empty(), nil
|
|
}
|
|
g, err := overlay.From(nodes, overlayCIDR(), "")
|
|
if err != nil && len(refused) > 0 {
|
|
// The network is missing something, and some machines could not be resolved at all. Those
|
|
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
|
// reporting "no hub" would name a consequence and hide the cause.
|
|
var who []string
|
|
for name, why := range refused {
|
|
who = append(who, fmt.Sprintf(" %s: %s", name, why))
|
|
}
|
|
sort.Strings(who)
|
|
return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+
|
|
"probably why:\n%s", err, len(refused), strings.Join(who, "\n"))
|
|
}
|
|
return g, err
|
|
}
|
|
|
|
// graph is the whole mesh's network, for showing it.
|
|
func graph(ctx context.Context, inv *inventory.Inventory) ([]overlay.Node, overlay.Graph, error) {
|
|
on, refused, err := whoResolves(ctx, inv, overlay.Requirement)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
g, err := network(ctx, inv, on, refused)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return g.Nodes(), g.Graph(), nil
|
|
}
|
|
|
|
// whoResolves is the machines whose resolution answers a requirement, and why the others did not.
|
|
//
|
|
// By what a module **provides**, not by its name. WireGuard is one way to have a private network
|
|
// and there could be others, so a machine is on the network because something it runs provides
|
|
// one — asking for a particular module by name would be the mistake this whole mechanism exists
|
|
// to avoid.
|
|
//
|
|
// Resolved rather than read from the assignment table, because a module can arrive by being
|
|
// required by something else, and a machine that needs the private network to do its job is on it
|
|
// for the same reason as one that was handed it directly.
|
|
func whoResolves(ctx context.Context, inv *inventory.Inventory, requirement string) (
|
|
map[string]bool, map[string]string, error) {
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
on := map[string]bool{}
|
|
// Why a node could not be resolved, kept rather than raised: one broken node must not stop
|
|
// the rest being described, and whoever is rendering that node will raise it themselves.
|
|
refused := map[string]string{}
|
|
for _, n := range nodes {
|
|
plan, _, err := planFor(ctx, inv, n.Name)
|
|
if err != nil {
|
|
refused[n.Name] = err.Error()
|
|
continue
|
|
}
|
|
for _, m := range plan.Modules {
|
|
for _, offered := range m.Offers() {
|
|
if offered == requirement {
|
|
on[n.Name] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return on, refused, nil
|
|
}
|
|
|
|
// rendering is everything a declaration needs, computed over the whole mesh.
|
|
func generators(ctx context.Context, inv *inventory.Inventory) (
|
|
map[string]catalogue.Generator, error) {
|
|
on, refused, err := whoResolves(ctx, inv, overlay.Addressing)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
net, err := network(ctx, inv, on, refused)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Both generators see the same machines: the ones on the private network. Names for a machine
|
|
// that is not on it would resolve to addresses it cannot reach, which is worse than no names.
|
|
return map[string]catalogue.Generator{
|
|
overlay.Name: net,
|
|
overlay.Names: overlay.NamesFor(net.Nodes()),
|
|
}, nil
|
|
}
|
|
|
|
func overlayShow(ctx context.Context, inv *inventory.Inventory) error {
|
|
nodes, computed, err := graph(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(nodes) == 0 {
|
|
// Not "this mesh has no nodes", which it said until the network became a module and was
|
|
// then a lie about the cause: a mesh can have every node it will ever have and nobody on
|
|
// the private network, because nobody asked for one.
|
|
fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n",
|
|
overlay.Name)
|
|
return nil
|
|
}
|
|
|
|
for _, n := range nodes {
|
|
place := n.Address
|
|
if place == "" {
|
|
// Said, not skipped. A node with no place is a node with no network, and it should
|
|
// be visible here rather than quietly absent from a list of who is on it.
|
|
place = "no address — run `overlay place`"
|
|
}
|
|
fmt.Printf("%-16s %-14s", n.Name, place)
|
|
switch {
|
|
case n.Hub:
|
|
fmt.Print(" hub")
|
|
case !n.Reachable():
|
|
fmt.Print(" not dialable")
|
|
}
|
|
if n.Site != "" {
|
|
fmt.Printf(" at %s", n.Site)
|
|
}
|
|
fmt.Println()
|
|
for _, p := range computed[n.Name] {
|
|
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SilentFor is how long a node may be quiet before the mesh says so.
|
|
//
|
|
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
|
// is not the point — being able to say "out of touch" at all is, and nothing could before.
|
|
const SilentFor = 3 * time.Minute
|
|
|
|
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
|
//
|
|
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
|
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
|
// picture of the mesh.
|
|
func heardFrom(n inventory.Node) string {
|
|
silent, ever := n.Silent()
|
|
switch {
|
|
case !ever:
|
|
return "never spoken"
|
|
case silent > SilentFor:
|
|
return "out of touch " + roughly(silent)
|
|
default:
|
|
return "here"
|
|
}
|
|
}
|
|
|
|
// roughly is a duration a person reads rather than parses.
|
|
func roughly(d time.Duration) string {
|
|
switch {
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%dh", int(d.Hours()))
|
|
default:
|
|
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
|
}
|
|
}
|
|
|
|
func moduleCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 {
|
|
return errors.New("module add <file>, module list, or module forget <name>")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
switch args[0] {
|
|
case "add":
|
|
set := flag.NewFlagSet("module add", flag.ContinueOnError)
|
|
repo := set.String("source", "", "where this module comes from")
|
|
ref := set.String("ref", "", "the branch followed there")
|
|
commit := set.String("commit", "", "the commit this manifest was read at")
|
|
positionals, err := parseAround(set, args[1:])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("module add <manifest.json> [--source <repo> --ref <branch> --commit <sha>]")
|
|
}
|
|
raw, err := os.ReadFile(positionals[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m, err := catalogue.ParseManifest(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Provenance together or not at all. A source with no commit cannot be compared against
|
|
// anything, so it would record where the module came from and still never be able to say
|
|
// the mesh is behind it — which is the one thing recording it is for.
|
|
if (*repo == "") != (*commit == "") {
|
|
return errors.New("--source and --commit go together: a source with no commit " +
|
|
"cannot be compared against anything, and a commit with no source has nothing " +
|
|
"to be compared with")
|
|
}
|
|
if err := inv.RegisterModule(ctx, m, inventory.Source{
|
|
Repository: *repo, Ref: *ref, BuiltFrom: *commit,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s registered", m.Module)
|
|
if *commit != "" {
|
|
fmt.Printf(" from %s", short(*commit))
|
|
}
|
|
if len(m.Provides) > 0 {
|
|
fmt.Printf(", providing %s", describeOffers(m.Provides))
|
|
}
|
|
fmt.Println()
|
|
for _, c := range m.Claims {
|
|
fmt.Printf(" claims %s, one per %s\n", c.Name, c.At())
|
|
}
|
|
return nil
|
|
|
|
case "list":
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(shelf) == 0 {
|
|
fmt.Println("this mesh knows about no modules yet")
|
|
return nil
|
|
}
|
|
var names []string
|
|
for n := range shelf {
|
|
names = append(names, n)
|
|
}
|
|
sort.Strings(names)
|
|
for _, n := range names {
|
|
m := shelf[n]
|
|
fmt.Printf("%-20s", m.Module)
|
|
if len(m.Provides) > 0 {
|
|
fmt.Printf(" provides %s", describeOffers(m.Provides))
|
|
}
|
|
for _, c := range m.Claims {
|
|
fmt.Printf(" claims %s/%s", c.At(), c.Name)
|
|
}
|
|
fmt.Println()
|
|
}
|
|
return nil
|
|
|
|
case "moved":
|
|
if len(args) != 3 {
|
|
return errors.New("module moved <name> <commit> — the source has a newer commit")
|
|
}
|
|
if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil {
|
|
return err
|
|
}
|
|
from, err := inv.SourceOf(ctx, args[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if from.Current() {
|
|
fmt.Printf("%s is current at %s\n", args[1], short(from.Head))
|
|
return nil
|
|
}
|
|
fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
|
|
args[1], short(from.BuiltFrom), short(from.Head))
|
|
fmt.Println(" build it and `module add` the result to catch up")
|
|
return nil
|
|
|
|
case "forget":
|
|
if len(args) != 2 {
|
|
return errors.New("module forget <name>")
|
|
}
|
|
if err := inv.ForgetModule(ctx, args[1]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s forgotten\n", args[1])
|
|
return nil
|
|
|
|
default:
|
|
return fmt.Errorf("module has no %q; it has add, list, moved and forget", args[0])
|
|
}
|
|
}
|
|
|
|
func assignCommand(ctx context.Context, verb string, args []string) error {
|
|
if len(args) != 2 {
|
|
return fmt.Errorf("%s <node> <module>", verb)
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
if verb == "unassign" {
|
|
if err := inv.Unassign(ctx, args[0], args[1]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s no longer runs %s — run `push %s` to make it so\n", args[0], args[1], args[0])
|
|
return nil
|
|
}
|
|
if err := inv.Assign(ctx, args[0], args[1]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s is assigned %s\n", args[0], args[1])
|
|
|
|
// Resolved immediately, because an assignment that cannot be applied should be said now
|
|
// rather than at the next push. The assignment is kept either way: it is what a person meant,
|
|
// and the refusal is about the set rather than about this one.
|
|
if _, _, err := planFor(ctx, inv, args[0]); err != nil {
|
|
fmt.Println()
|
|
return err
|
|
}
|
|
fmt.Printf(" run `push %s` to send it\n", args[0])
|
|
return nil
|
|
}
|
|
|
|
// planFor works out everything a node should run, from what was assigned to it.
|
|
func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
assigned, err := inv.Assigned(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
capabilities, err := inv.ProfileOf(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
var site string
|
|
for _, p := range places {
|
|
if p.Name == nodeName {
|
|
site = p.Site
|
|
}
|
|
}
|
|
|
|
world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
world.Pinned, err = inv.PinsFor(ctx, nodeName)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
resolved, err := catalogue.Resolve(shelf, assigned,
|
|
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
|
|
At: onNetwork[nodeName]}, world)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
|
|
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
|
|
// password a provider is told to create is the one its consumer was given — and sealed to
|
|
// this node before it was ever written down, so nothing between here and there can read it.
|
|
for i, n := range resolved.Needs {
|
|
secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.From)
|
|
if err != nil {
|
|
// Said rather than skipped. A machine that resolves cleanly and receives no
|
|
// credential is one that will fail to authenticate at some later, less obvious
|
|
// moment.
|
|
return catalogue.Resolution{}, nil, fmt.Errorf(
|
|
"%s needs %s from %s and no credential could be made for it: %w",
|
|
nodeName, n.Name, n.From, err)
|
|
}
|
|
resolved.Needs[i].Sealed = secret.ForConsumer
|
|
}
|
|
|
|
// Settings for everything that resolved, including modules nobody assigned directly: a
|
|
// requirement pulled in by something else is still configurable, and finding out that it is
|
|
// not only when you try would be an arbitrary line nobody could predict.
|
|
settings := catalogue.SettingsBy{}
|
|
var stray []string
|
|
for _, m := range resolved.Modules {
|
|
layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
|
|
if err != nil {
|
|
return catalogue.Resolution{}, nil, err
|
|
}
|
|
if len(layers) == 0 {
|
|
continue
|
|
}
|
|
settings[m.Module] = layers
|
|
stray = append(stray, catalogue.UnusedSettings(m, layers)...)
|
|
}
|
|
if len(stray) > 0 {
|
|
// Somebody set something that reaches no file. Said here rather than discovered by the
|
|
// machine not behaving differently, which is the slowest way there is.
|
|
return catalogue.Resolution{}, nil, fmt.Errorf(
|
|
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))
|
|
}
|
|
return resolved, settings, nil
|
|
}
|
|
|
|
// theRestOfTheMesh is what every other node holds and offers.
|
|
//
|
|
// Two things at once because they come from the same place — resolving the other nodes — and
|
|
// because both are facts about what is actually running rather than records that could disagree
|
|
// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node
|
|
// runs; neither is a table somebody keeps up to date.
|
|
//
|
|
// **Two passes over the others.** What a node offers the mesh needs that node resolved, and
|
|
// resolving it may need what the mesh offers. So the first pass takes brokered requirements on
|
|
// trust and answers only *what does each node offer*; the second answers everything with that in
|
|
// hand. Nothing is ever declared from the first.
|
|
func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|
shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
|
|
|
// Every node, not only the placed ones. A machine that was never put on the private network
|
|
// still runs modules, still holds claims, and still offers whatever it offers.
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
siteOf := map[string]string{}
|
|
for _, p := range places {
|
|
siteOf[p.Name] = p.Site
|
|
}
|
|
// Which machines are actually on the private network, and what they are called there. Not
|
|
// "has an address" — that was true of every placed machine and told you nothing about whether
|
|
// anything could reach it. It is what resolved the module.
|
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
|
|
type candidate struct {
|
|
node catalogue.Node
|
|
assigned []string
|
|
}
|
|
var others []candidate
|
|
for _, n := range nodes {
|
|
if n.Name == exclude {
|
|
continue
|
|
}
|
|
theirs, err := inv.Assigned(ctx, n.Name)
|
|
if err != nil || len(theirs) == 0 {
|
|
continue
|
|
}
|
|
caps, _ := inv.ProfileOf(ctx, n.Name)
|
|
others = append(others, candidate{
|
|
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
|
|
At: onNetwork[n.Name]}, theirs})
|
|
}
|
|
|
|
offered := map[string][]catalogue.Provider{}
|
|
for _, o := range others {
|
|
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
|
|
if err != nil {
|
|
// Their set does not resolve for some other reason. Not this node's problem to
|
|
// report, and nothing of theirs is running, so it offers nothing.
|
|
continue
|
|
}
|
|
for _, m := range got.Modules {
|
|
for _, name := range m.OffersAt(catalogue.ScopeMesh) {
|
|
// What that module says a consumer needs to know, with that node's settings on
|
|
// it: a port somebody moved on the provider is a port its consumers must be told
|
|
// about, and the two coming from different places is how they come to disagree.
|
|
serves := m.Serves[name]
|
|
if len(serves) > 0 {
|
|
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
serves, err = catalogue.Settle(serves, layers)
|
|
if err != nil {
|
|
return catalogue.World{}, err
|
|
}
|
|
}
|
|
offered[name] = append(offered[name], catalogue.Provider{
|
|
Node: o.node.Name, At: o.node.At, Serves: serves})
|
|
}
|
|
}
|
|
}
|
|
for k := range offered {
|
|
sort.Slice(offered[k], func(i, j int) bool {
|
|
return offered[k][i].Node < offered[k][j].Node
|
|
})
|
|
}
|
|
|
|
world := catalogue.World{Offered: offered}
|
|
for _, o := range others {
|
|
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
world.Held = append(world.Held, got.Claims...)
|
|
}
|
|
return world, nil
|
|
}
|
|
|
|
// whereEveryoneIs is each machine's name on the private network, for the ones on it.
|
|
//
|
|
// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every
|
|
// other path here answers a question about one node by resolving the others; this one is called
|
|
// *from* that path, so doing the same would not terminate — which it did not, for two minutes,
|
|
// until it was run.
|
|
//
|
|
// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the
|
|
// private network depends on what it was assigned and what that requires, both of which are local
|
|
// facts. What it takes *from* other machines does not change the answer.
|
|
//
|
|
// The distinction that matters is kept: a machine absent from the network module's own view is
|
|
// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the
|
|
// network became something a machine is given.
|
|
func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
|
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
|
|
|
places, err := inv.Overlays(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[string]string{}
|
|
for _, p := range places {
|
|
if p.Address == "" {
|
|
continue
|
|
}
|
|
assigned, err := inv.Assigned(ctx, p.Name)
|
|
if err != nil || len(assigned) == 0 {
|
|
continue
|
|
}
|
|
caps, _ := inv.ProfileOf(ctx, p.Name)
|
|
got, err := catalogue.Resolve(shelf, assigned,
|
|
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
|
|
catalogue.World{Unchecked: true})
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, m := range got.Modules {
|
|
for _, offered := range m.Offers() {
|
|
if offered == overlay.Requirement {
|
|
out[p.Name] = overlay.InternalName(p.Name)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// declarationFor is everything a node would be sent.
|
|
//
|
|
// One place, because there were three and one of them was written before credentials existed and
|
|
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
|
// `plan --json` handed to anything reading it.
|
|
func declarationFor(ctx context.Context, inv *inventory.Inventory, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) {
|
|
gens, err := generators(ctx, inv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return declarationWith(ctx, inv, node, plan, settings, gens)
|
|
}
|
|
|
|
// declarationWith is the same, for a caller that has already worked out the generators once and
|
|
// is about to use them for every node.
|
|
func declarationWith(ctx context.Context, inv *inventory.Inventory, node string,
|
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
|
gens map[string]catalogue.Generator) ([]map[string]any, error) {
|
|
grants, err := grantsFor(ctx, inv, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return plan.Declaration(
|
|
catalogue.Rendering{Settings: settings, Generators: gens, Grants: grants})
|
|
}
|
|
|
|
// grantsFor is every credential this node must create, because something elsewhere uses it.
|
|
//
|
|
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
|
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
|
// the mesh hands over something it cannot itself use.
|
|
func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.Grant, error) {
|
|
issued, err := inv.SecretsFrom(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
|
// from a record beside it. A provider told to create a password and not what to create it for
|
|
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
|
out := make([]catalogue.Grant, 0, len(issued))
|
|
for _, s := range issued {
|
|
plan, settings, err := planFor(ctx, inv, s.Consumer)
|
|
if err != nil {
|
|
// Their set does not resolve. Skipped rather than fatal: this node is not the place
|
|
// to report another machine's problem, and a grant for something that is not going to
|
|
// run would have the provider create a user nothing uses.
|
|
continue
|
|
}
|
|
from, values, err := plan.ContributionsTo(s.Name, settings)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, catalogue.Grant{
|
|
Provision: s.Name, Consumer: s.Consumer,
|
|
From: from, Values: values, Sealed: s.ForProvider})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func planCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("plan", flag.ContinueOnError)
|
|
// Because "one resource" does not tell you whether the settings landed. Being able to read
|
|
// the file before it is sent is the difference between believing a merge worked and knowing.
|
|
show := set.Bool("files", false, "print the files this node would be given")
|
|
// The declaration exactly as the node would receive it. For handing to something else --
|
|
// checking it against the host's own parser, most usefully, which is the only way to know
|
|
// that what the control plane emits is what the host accepts.
|
|
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("plan <node> [--files] [--json]")
|
|
}
|
|
args = positionals
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
plan, settings, err := planFor(ctx, inv, args[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(plan.Modules) == 0 {
|
|
fmt.Printf("%s is assigned nothing\n", args[0])
|
|
return nil
|
|
}
|
|
if *asJSON {
|
|
resources, err := declarationFor(ctx, inv, args[0], plan, settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body, err := json.MarshalIndent(
|
|
map[string]any{"declaration": 1, "resources": resources}, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(string(body))
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("%s would run:\n", args[0])
|
|
for _, m := range plan.Modules {
|
|
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
|
}
|
|
for _, c := range plan.Claims {
|
|
fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope)
|
|
}
|
|
// What this machine depends on that is not on it. Worth saying out loud: it is the only part
|
|
// of a node's set that stops working when a *different* machine goes away, and nothing else
|
|
// in this output would have told anybody that.
|
|
for _, n := range plan.Needs {
|
|
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
|
}
|
|
resources, err := declarationFor(ctx, inv, args[0], plan, settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for module, layers := range settings {
|
|
for _, layer := range layers {
|
|
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
|
}
|
|
}
|
|
fmt.Printf("\n%d resource(s)\n", len(resources))
|
|
|
|
if *show {
|
|
for _, r := range resources {
|
|
content, ok := r["content"].(string)
|
|
if !ok {
|
|
continue
|
|
}
|
|
fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// pushCommand sends nodes everything they should be: their place on the network, and what their
|
|
// assignments resolve to.
|
|
//
|
|
// One declaration, not two. A node holding its network and not its modules, or the reverse, is
|
|
// half-configured for as long as that lasts — and the two are computed from the same picture of
|
|
// the mesh, so sending them apart would let them disagree.
|
|
func pushCommand(ctx context.Context, args []string) error {
|
|
if len(args) > 1 {
|
|
return errors.New("push [<node>] — one node, or all of them")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
ident, err := openIdentity(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer ident.Close()
|
|
|
|
// Every node, not only the ones on the private network. A machine that was never given the
|
|
// network module still takes modules, and iterating the network here is what used to make
|
|
// "on the network" and "managed" the same thing.
|
|
nodes, err := inv.Nodes(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gens, err := generators(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
server, err := link.Connect(nil, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer server.Close()
|
|
|
|
// Every node is resolved before anything is sent. A push that configured three nodes and then
|
|
// refused on the fourth would leave the mesh in a state nobody asked for, and the fourth is
|
|
// exactly where a claim collision shows up.
|
|
type ready struct {
|
|
node string
|
|
resources []map[string]any
|
|
}
|
|
var sending []ready
|
|
var refusals []string
|
|
|
|
for _, n := range nodes {
|
|
if len(args) == 1 && n.Name != args[0] {
|
|
continue
|
|
}
|
|
plan, settings, err := planFor(ctx, inv, n.Name)
|
|
if err != nil {
|
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err))
|
|
continue
|
|
}
|
|
// The private network is in here with everything else. It used to be composed separately
|
|
// and prepended, which meant every machine with an address was on it and no machine could
|
|
// be kept off. It is a module now, so it arrives the way a module does.
|
|
resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens)
|
|
if err != nil {
|
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err))
|
|
continue
|
|
}
|
|
if len(resources) == 0 {
|
|
fmt.Printf("%s is assigned nothing — skipped\n", n.Name)
|
|
continue
|
|
}
|
|
sending = append(sending, ready{n.Name, resources})
|
|
}
|
|
|
|
if len(refusals) > 0 {
|
|
return fmt.Errorf("nothing was sent. %d node(s) could not be resolved:\n\n%s",
|
|
len(refusals), strings.Join(refusals, "\n\n"))
|
|
}
|
|
|
|
for _, s := range sending {
|
|
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
|
|
}
|
|
fmt.Printf("\n%d node(s) told\n", len(sending))
|
|
return nil
|
|
}
|
|
|
|
// short is a commit as a person refers to it.
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// statusCommand answers "did my change go out?".
|
|
//
|
|
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
|
|
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
|
|
// this is worse to live with whatever its other properties.
|
|
//
|
|
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
|
|
// source now has, and which machines are running the old one.
|
|
func statusCommand(ctx context.Context) error {
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
behind, err := inv.Behind(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(behind) == 0 {
|
|
fmt.Println("every module with a source is built from what that source has")
|
|
return nil
|
|
}
|
|
|
|
var names []string
|
|
for m := range behind {
|
|
names = append(names, m)
|
|
}
|
|
sort.Strings(names)
|
|
|
|
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
|
|
for _, m := range names {
|
|
from, err := inv.SourceOf(ctx, m)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf(" %-20s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
|
|
if nodes := behind[m]; len(nodes) > 0 {
|
|
// The part somebody actually wants. A module being out of date is a fact about the
|
|
// catalogue; machines running the old one is the thing with consequences.
|
|
fmt.Printf(" %-20s running on %s\n", "", strings.Join(nodes, ", "))
|
|
} else {
|
|
fmt.Printf(" %-20s assigned to nothing\n", "")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// parseAround reads flags that may sit before, after or between positional arguments.
|
|
//
|
|
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
|
|
// parses no flags at all and silently ignores every one of them. The host learned this the same
|
|
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
|
|
// keeps naming, and it looks exactly like success.
|
|
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
|
var positionals []string
|
|
rest := args
|
|
for {
|
|
if err := set.Parse(rest); err != nil {
|
|
return nil, err
|
|
}
|
|
rest = set.Args()
|
|
if len(rest) == 0 {
|
|
return positionals, nil
|
|
}
|
|
positionals = append(positionals, rest[0])
|
|
rest = rest[1:]
|
|
}
|
|
}
|
|
|
|
func settingsCommand(ctx context.Context, args []string) error {
|
|
if len(args) == 0 {
|
|
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
set := flag.NewFlagSet("settings", flag.ContinueOnError)
|
|
node := set.String("node", "", "one machine, rather than the whole mesh")
|
|
positionals, err := parseAround(set, args[1:])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
where := "the whole mesh"
|
|
if *node != "" {
|
|
where = *node
|
|
}
|
|
|
|
switch args[0] {
|
|
case "set":
|
|
if len(positionals) != 2 {
|
|
return errors.New("settings set <module> <settings.json> [--node <node>]")
|
|
}
|
|
raw, err := os.ReadFile(positionals[1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var values map[string]any
|
|
if err := json.Unmarshal(raw, &values); err != nil {
|
|
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
|
|
}
|
|
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
|
return err
|
|
}
|
|
|
|
var keys []string
|
|
for k := range values {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
|
|
fmt.Println(" run `push` to send it")
|
|
return nil
|
|
|
|
case "clear":
|
|
if len(positionals) != 1 {
|
|
return errors.New("settings clear <module> [--node <node>]")
|
|
}
|
|
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
|
return nil
|
|
|
|
default:
|
|
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
|
|
}
|
|
}
|
|
|
|
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
|
|
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
|
|
// entirely different things about where the answer has to be.
|
|
func describeOffers(offers []catalogue.Offer) string {
|
|
var out []string
|
|
for _, o := range offers {
|
|
if o.At() == catalogue.ScopeMesh {
|
|
out = append(out, o.Name+" (from anywhere in the mesh)")
|
|
continue
|
|
}
|
|
out = append(out, o.Name)
|
|
}
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
// pinCommand says which node a machine gets a provision from.
|
|
//
|
|
// Needed only when more than one could answer, and recordable before that -- a mesh with one
|
|
// database should not change where an existing machine gets its data the day a second one
|
|
// arrives.
|
|
func pinCommand(ctx context.Context, args []string, setting bool) error {
|
|
if setting && len(args) != 3 {
|
|
return errors.New("pin <node> <provision> <from-node>")
|
|
}
|
|
if !setting && len(args) != 2 {
|
|
return errors.New("unpin <node> <provision>")
|
|
}
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
if !setting {
|
|
if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
|
|
return nil
|
|
}
|
|
if args[0] == args[2] {
|
|
// Allowed by nothing here, and worth saying rather than resolving into a confusing
|
|
// refusal later: a node providing something to itself is a node-scoped provision, and
|
|
// this field is for the other kind.
|
|
return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+
|
|
"provides, which does not need saying", args[0], args[1])
|
|
}
|
|
if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2])
|
|
fmt.Printf(" run `push %s` to send it\n", args[0])
|
|
return nil
|
|
}
|
|
|
|
// buildCommand builds a module from its source and records what came out.
|
|
//
|
|
// **Run where there is a container runtime**, which is why it is a command rather than something
|
|
// the control plane does on its own: building needs to run things on a machine, and what the
|
|
// control plane may send a machine is bounded by the declaration language. This is the shape the
|
|
// builder module will take when it is given work over the broker; today a person runs it, and the
|
|
// mesh records the result the same way either way.
|
|
func buildCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
|
ref := set.String("ref", "", "the branch, tag or commit to build")
|
|
registry := set.String("registry", os.Getenv("MESH_REGISTRY"),
|
|
"host:port of the registry to publish to")
|
|
workspace := set.String("workspace", os.TempDir(), "where to clone and build")
|
|
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("build <repository> [--ref R] [--registry host:port]")
|
|
}
|
|
if strings.TrimSpace(*registry) == "" {
|
|
return errors.New(
|
|
"no --registry and no MESH_REGISTRY: a built artifact nobody can fetch is not built")
|
|
}
|
|
|
|
publisher := builder.Registry{Address: *registry, Run: builder.Command}
|
|
result, err := builder.Build(ctx, builder.Command, publisher, positionals[0], *ref, *workspace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, made := range result.Built {
|
|
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
|
}
|
|
if *dryRun {
|
|
body, err := json.MarshalIndent(result.Manifest, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(string(body))
|
|
return nil
|
|
}
|
|
|
|
inv, err := openInventory(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inv.Close()
|
|
|
|
// Recorded with where it came from, so "is this current?" is answerable without building it
|
|
// again (novox/hq ADR 0009).
|
|
if err := inv.RegisterModule(ctx, result.Manifest, inventory.Source{
|
|
Repository: positionals[0], Ref: *ref, BuiltFrom: result.Commit, Head: result.Commit,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("\n%s %s, built from %s\n",
|
|
result.Manifest.Module, result.Manifest.Version, short(result.Commit))
|
|
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", result.Manifest.Module)
|
|
return nil
|
|
}
|