From 86236137041d1b6206453e6f5683a76dee4aaef3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 31 Aug 2026 13:30:54 +0200 Subject: [PATCH] Split main.go along the seams it already had MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2,769 lines and 59 functions, holding command parsing, store opening, resolution, the board, rotation, licences, builds and status rendering. Nothing in it was wrong. It grew because appending was always the cheapest next step, and no single edit was the one that should have been a new file. That is exactly how novox/hq ADR 0001 records `hal/sdk` reaching 155 files and 34,636 lines — "containing code from every context", with each addition avoiding a cycle and none of them the mistake. This is the same shape at 8% of the size, which is why it is worth doing now rather than noting. Eight files, along boundaries that already existed: what a machine is; the private network; the catalogue; working out what one machine should be; sending it; builds; the three questions; and reaching each context's store. main.go keeps what a main is for — parsing arguments and dispatching. A pure move. No behaviour changed, no test changed, and the gate is green before and after — which is the only thing that makes a refactor this size safe to do in one commit. --- cmd/mesh-control/build.go | 438 ++++++ cmd/mesh-control/main.go | 2581 ----------------------------------- cmd/mesh-control/modules.go | 350 +++++ cmd/mesh-control/network.go | 358 +++++ cmd/mesh-control/nodes.go | 296 ++++ cmd/mesh-control/plan.go | 556 ++++++++ cmd/mesh-control/push.go | 409 ++++++ cmd/mesh-control/status.go | 218 +++ cmd/mesh-control/stores.go | 113 ++ 9 files changed, 2738 insertions(+), 2581 deletions(-) create mode 100644 cmd/mesh-control/build.go create mode 100644 cmd/mesh-control/modules.go create mode 100644 cmd/mesh-control/network.go create mode 100644 cmd/mesh-control/nodes.go create mode 100644 cmd/mesh-control/plan.go create mode 100644 cmd/mesh-control/push.go create mode 100644 cmd/mesh-control/status.go create mode 100644 cmd/mesh-control/stores.go diff --git a/cmd/mesh-control/build.go b/cmd/mesh-control/build.go new file mode 100644 index 0000000..97bf4c0 --- /dev/null +++ b/cmd/mesh-control/build.go @@ -0,0 +1,438 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-control/internal/broker" + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/link" +) + +// asking a build machine for a module, and what came back. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// buildCommand builds a module from its source and records what came out. +// +// **Run where there is a container runtime**, which is why it is a command rather than something +// the control plane does on its own: building needs to run things on a machine, and what the +// control plane may send a machine is bounded by the declaration language. This is the shape the +// builder module will take when it is given work over the broker; today a person runs it, and the +// mesh records the result the same way either way. +func buildCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("build", flag.ContinueOnError) + ref := set.String("ref", "", "the branch, tag or commit to build") + wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer") + dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing") + // Every module whose source has moved, rather than one named repository. + // + // **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh + // already knows which modules are behind their source, so making a person read that list and + // retype each repository is asking them to be the loop. Naming a repository and asking which + // ones need building are different requests, so they are not combined. + behind := set.Bool("behind", false, "every module the mesh holds older than its source has") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if *behind { + if len(positionals) != 0 { + return errors.New("build or build --behind, not both: one names a " + + "repository and the other asks which need building") + } + return buildBehind(ctx, *wait) + } + if len(positionals) != 1 { + return errors.New("build [--ref R] [--wait D] [--dry-run]") + } + + if *dryRun { + return buildAndShow(ctx, positionals[0], *ref, *wait) + } + return buildOne(ctx, positionals[0], *ref, *wait) +} + +// buildFrom turns what a builder said into what the mesh keeps. +func buildFrom(result link.BuildResult) inventory.Build { + kept := inventory.Build{ + ID: result.ID, Repository: result.Repository, Ref: result.Ref, + Commit: result.Commit, On: result.On, Failed: result.Failed, + } + for _, made := range result.Made { + kept.Made = append(kept.Made, inventory.Artifact{ + Name: made.Name, Kind: made.Kind, Reference: made.Reference, + }) + } + // The module name comes from the manifest, which only exists when the build got that far. + if len(result.Manifest) > 0 { + if m, err := catalogue.ParseManifest(result.Manifest); err == nil { + kept.Module = m.Module + } + } + return kept +} + +// buildsCommand says what has been built lately. +func buildsCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("builds", flag.ContinueOnError) + limit := set.Int("n", 20, "how many to show") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + module := "" + if len(positionals) == 1 { + module = positionals[0] + } else if len(positionals) > 1 { + return errors.New("builds [] [-n N]") + } + + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + builds, err := inv.Builds(ctx, module, *limit) + if err != nil { + return err + } + if len(builds) == 0 { + // Said rather than printed as nothing: an empty list and a failed read must never look + // the same, and getting here means the store answered. + if module != "" { + fmt.Printf("nothing has been built for %s\n", module) + return nil + } + fmt.Println("nothing has been built yet") + return nil + } + + for _, b := range builds { + what := b.Module + if what == "" { + // It failed before knowing what it was building, which is most of the interesting + // failures. The repository is what a person has to go and look at. + what = "?" + } + outcome := "built " + short(b.Commit) + if !b.Worked() { + outcome = "failed" + } + fmt.Printf("%-18s %-14s %-10s %s\n", + what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04")) + fmt.Printf(" %s", b.Repository) + if b.Ref != "" { + fmt.Printf(" at %s", b.Ref) + } + fmt.Println() + for _, made := range b.Made { + fmt.Printf(" %-10s %s\n", made.Kind, made.Reference) + } + if !b.Worked() { + // The builder's own first line. The whole failure is often a build log, and printing + // it here would bury every other row. + fmt.Printf(" %s\n", firstLine(b.Failed)) + } + } + return nil +} + +// builderCommand issues a build machine its own broker credential. +// +// **A build machine is not a node**, and giving it a node's account would let it read another +// machine's declarations. This is narrower and different: read the build queue, write the +// exchange and an asker's reply queue, and nothing else. +// +// Issued rather than assumed, because until this the builder used whatever credential it was +// handed — which in practice meant the broker's own administrative one. A program documented as +// holding its own credential and given somebody else's is worse than one with no story at all. +func builderCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("builder issue", flag.ContinueOnError) + // Which machine will use it. Given, the credential is delivered by the mesh rather than + // printed for somebody to carry — which is the difference between the builder being a module + // and being a program somebody configures. + forNode := set.String("node", "", + "the machine that will run it, so the mesh delivers the credential instead of printing it") + module := set.String("module", "builder", "the module on that machine that will read it") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 2 || positionals[0] != "issue" { + return errors.New("builder issue [--node ]") + } + name := positionals[1] + + management, err := broker.ManagementFromEnvironment() + if err != nil { + return err + } + + // The same shape of secret a token carries: enough entropy that guessing is not a strategy, + // and safe to put in a URL because that is where it goes. + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return err + } + password := base64.RawURLEncoding.EncodeToString(raw) + if err := management.CreateBuilderAccount(ctx, name, password); err != nil { + return err + } + + fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n", + name, link.BuildQueue, link.Exchange) + + if *forNode != "" { + known, err := broker.FromEnvironment() + if err != nil { + return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err) + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + // The URL and what verifies the broker, together. A mesh's broker presents a certificate + // of the mesh's own, which is in no public trust store — so a URL on its own reaches only + // a broker somebody else vouches for, and the connection fails at TLS with an error about + // an unknown authority rather than about a missing pin. + // + // **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same + // way: out of band relative to the broker, so what is trusted does not come from the thing + // being trusted. + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + }{ + URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, known.Address), + Fingerprint: known.Fingerprint, + }) + if err != nil { + return err + } + if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil { + return err + } + // Not printed. It is sealed to that machine and the mesh cannot read it back, which is + // the whole point — printing it here would put the one copy that matters on a terminal. + fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n", + *forNode, *module) + fmt.Printf(" run `push %s` to send it\n", *forNode) + return nil + } + + // The whole line only when the address is known. A URL with a placeholder where the host + // should be is a URL somebody pastes and then debugs, and the placeholder is the last thing + // they look at. + if known, err := broker.FromEnvironment(); err == nil { + fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address) + } else { + fmt.Printf(" the password is %s\n\n", password) + fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+ + " Put the password in MESH_BROKER_AMQP on the build machine.\n\n", + broker.AddressVar) + } + // Shown once, like a token, and for the same reason: what is stored is the broker's own hash + // of it, and a control plane that could show it back would be a control plane that holds it. + fmt.Println("This is the only time it is shown.") + return nil +} + +// buildBehind builds every module the mesh holds older than its source has. +// +// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already +// records where each module came from and what its source last had; until this, a person read +// that list and retyped each repository — which is a person being the loop, and the thing a +// pipeline was doing before it was taken away. +// +// Each is built and recorded on its own. **One failing does not stop the others**, for the same +// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad +// repository holds back nine good ones is a mesh where nobody dares add the tenth. +func buildBehind(ctx context.Context, wait time.Duration) error { + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + held, err := inv.Catalogued(ctx) + if err != nil { + return err + } + var stale []inventory.Entry + for _, e := range held { + if !e.Source.Current() { + stale = append(stale, e) + } + } + if len(stale) == 0 { + // Said rather than doing nothing quietly: "nothing needed building" and "this did not + // run" must never look the same. + fmt.Println("every module the mesh holds is what its source last had") + return nil + } + + fmt.Printf("%d module(s) behind their source:\n", len(stale)) + for _, e := range stale { + fmt.Printf(" %s %s < %s\n", + e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head)) + } + fmt.Println() + + var failed []string + for _, e := range stale { + fmt.Printf("--- %s\n", e.Manifest.Module) + // Its own recorded ref, not its head commit: a module tracking a branch should be built + // from that branch, and pinning to the commit the mesh happened to notice would quietly + // turn a tracked branch into a pin. + if err := buildOne(ctx, e.Source.Repository, e.Source.Ref, wait); err != nil { + fmt.Printf(" %v\n", err) + failed = append(failed, e.Manifest.Module) + } + } + + if len(failed) > 0 { + return fmt.Errorf("%d of %d could not be built: %s", + len(failed), len(stale), strings.Join(failed, ", ")) + } + fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n", + len(stale)) + return nil +} + +// buildOne asks a build machine for one repository and records everything that came back. +// +// Separated from the command so `--behind` can walk a list without a second path to the same act. +func buildOne(ctx context.Context, repository, ref string, wait time.Duration) error { + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + // Correlated by something the control plane makes, not by the module's name: two builds of one + // module can be in flight, and the second answer is not the first one's. + request := link.BuildRequest{ + ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), + Repository: repository, + Ref: ref, + } + fmt.Printf("asked for %s", request.Repository) + if ref != "" { + fmt.Printf(" at %s", ref) + } + fmt.Println() + + result, err := link.RequestBuild(ctx, server.Channel(), request, wait) + if err != nil { + return err + } + + // Kept before it is judged. A failed build that leaves no trace is indistinguishable from one + // nobody asked for, and the difference is the whole of whether somebody should be looking at + // something. + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil { + return err + } + + if result.Failed != "" { + // The builder's own words. Wrapping them in something about the control plane would put + // two explanations between a person and a build log. + return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) + } + + for _, made := range result.Made { + fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference) + } + + // Parsed with the same parser a hand-written manifest goes through. A second path would be a + // second thing to disagree about what a manifest is. + manifest, err := catalogue.ParseManifest(result.Manifest) + if err != nil { + return fmt.Errorf("%s built %s and what came back is not a manifest: %w", + result.On, result.Repository, err) + } + + // Recorded with where it came from, so "is this current?" is answerable without building it + // again (novox/hq ADR 0009). + if err := inv.RegisterModule(ctx, manifest, inventory.Source{ + Repository: result.Repository, Ref: result.Ref, + BuiltFrom: result.Commit, Head: result.Commit, + }); err != nil { + return err + } + fmt.Printf("\n%s %s, built on %s from %s\n", + manifest.Module, manifest.Version, result.On, short(result.Commit)) + fmt.Printf(" run `assign %s` to put it somewhere\n", manifest.Module) + return nil +} + +// buildAndShow builds and prints the manifest without recording anything. +func buildAndShow(ctx context.Context, repository, ref string, wait time.Duration) error { + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{ + ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Ref: ref, + }, wait) + if err != nil { + return err + } + if result.Failed != "" { + return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) + } + manifest, err := catalogue.ParseManifest(result.Manifest) + if err != nil { + return fmt.Errorf("%s built %s and what came back is not a manifest: %w", + result.On, result.Repository, err) + } + body, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + fmt.Println(string(body)) + return nil +} + +// answers is what the three questions came back with, read once. +type answers struct { + wrong []inventory.Doing + nodes []inventory.Node + quiet []inventory.Node + behind map[string][]string + sources map[string]inventory.Source + // waiting is every machine not running what the mesh would send it. + waiting []inventory.Machine +} diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 5327b1b..94440aa 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -8,30 +8,17 @@ package main import ( "context" - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" "flag" "fmt" "os" "os/signal" - "sort" - "strings" "syscall" - "time" - "github.com/novox/mesh-control/internal/broker" - "github.com/novox/mesh-control/internal/catalogue" "github.com/novox/mesh-control/internal/identity" "github.com/novox/mesh-control/internal/inventory" "github.com/novox/mesh-control/internal/licences" "github.com/novox/mesh-control/internal/link" - "github.com/novox/mesh-control/internal/overlay" "github.com/novox/mesh-control/internal/store" - "github.com/novox/mesh-control/internal/token" ) // version is stamped at link time. Unset in a development build, and it says so rather than @@ -174,1783 +161,6 @@ Each context reaches its own store through its own credential (novox/hq ADR 0008 fmt.Fprintln(os.Stderr) } -// migrate brings every held context's schema up to date. -// -// Reported per context and per migration, because this runs during a bootstrap on a machine with -// nothing else on it — the output is the only account of what happened, and "migrated" is not one. -func migrate(ctx context.Context) error { - for _, c := range held { - migrations, err := c.migrations() - if err != nil { - return err - } - - s, err := store.Open(ctx, c.name) - if err != nil { - return err - } - defer s.Close() - - // The bootstrap raises PostgreSQL moments before this runs, and a container that is - // running is not a database that will answer — a distinction this project has already - // paid for once, when a crash-looping database reported itself as up between restarts. - if err := s.Ready(ctx, 60*time.Second); err != nil { - return err - } - - done, err := s.Migrate(ctx, migrations) - for _, m := range done { - fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name) - } - if err != nil { - return err - } - if len(done) == 0 { - applied, err := s.AppliedMigrations(ctx) - if err != nil { - return err - } - fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied)) - } - } - - // The modules the control plane ships with itself. Recorded here rather than by hand, because - // a mesh whose own private network is missing from the catalogue would have nothing to assign - // and no way to say why. - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - for _, m := range provided { - if err := inv.Provide(ctx, m); err != nil { - return err - } - fmt.Printf("provided %s\n", m.Module) - } - return nil -} - -// provided is what comes with the control plane rather than from a repository. -// -// WireGuard, the names, and the domain module over both. The first two are here because the code -// that works out their files is here: -// a peer list is derived from every machine at once, so it cannot be written in a manifest, and -// whatever computes it has to live wherever the whole picture is. -// -// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent -// from a machine nobody gave it to. -func providedModules() []catalogue.Manifest { - var out []catalogue.Manifest - for _, raw := range []map[string]any{ - overlay.Manifest(), overlay.NamesManifest(), overlay.ResolverManifest(), - overlay.DomainManifest(), - } { - var m catalogue.Manifest - b, _ := json.Marshal(raw) - _ = json.Unmarshal(b, &m) - out = append(out, m) - } - return out -} - -var provided = providedModules() - -// openInventory connects and waits, the way every command that touches it needs to. -func openInventory(ctx context.Context) (*inventory.Inventory, error) { - inv, err := inventory.Open(ctx) - if err != nil { - return nil, err - } - if err := inv.Ready(ctx, 30*time.Second); err != nil { - inv.Close() - return nil, err - } - return inv, nil -} - -func nodeCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("node add , node list, or node show ") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - switch args[0] { - case "show": - if len(args) != 2 { - return errors.New("node show ") - } - return showNode(ctx, inv, args[1]) - case "add": - if len(args) != 2 { - return errors.New("node add ") - } - node, err := inv.AddNode(ctx, args[1]) - if err != nil { - return err - } - fmt.Printf("added %s (%s)\n", node.Name, node.ID) - return nil - - case "list": - nodes, err := inv.Nodes(ctx) - if err != nil { - return err - } - if len(nodes) == 0 { - // Said rather than printed as nothing: an empty list and a failed read must never - // look the same, and this command answering "none" is only honest because getting - // here means the store answered. - fmt.Println("this mesh has no node records yet") - return nil - } - for _, n := range nodes { - fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID) - } - return nil - - default: - return fmt.Errorf("node has no %q; it has add and list", args[0]) - } -} - -func tokenCommand(ctx context.Context, args []string) error { - if len(args) == 0 || args[0] != "issue" { - return errors.New("token issue --node , or token issue --new ") - } - - set := flag.NewFlagSet("token issue", flag.ContinueOnError) - existing := set.String("node", "", "issue for a node record that already exists") - fresh := set.String("new", "", "create the node record, then issue for it") - validFor := set.Duration("for", time.Hour, "how long the token may be used") - if err := set.Parse(args[1:]); err != nil { - return err - } - - // Exactly one, because the difference is what the token binds to. A command that guessed - // would sometimes create a second record for a machine that already has one. - if (*existing == "") == (*fresh == "") { - return errors.New("give exactly one of --node or --new : the first is a " + - "machine the mesh already has a record for, the second is one it has never seen") - } - - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - name := *existing - if *fresh != "" { - node, err := inv.AddNode(ctx, *fresh) - if err != nil { - return err - } - name = node.Name - } - - issued, err := inv.IssueToken(ctx, name, *validFor) - if err != nil { - return err - } - - // Assembled from two contexts by the process that holds both grants. Neither reads the - // other's store (novox/hq ADR 0008) — each is asked for its own part. - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - key, err := ident.Establish(ctx) - if err != nil { - return err - } - - // The account is created before the token is handed over, which is what removes the - // chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can - // exist before it does. The one-time secret IS the password, so a node's first connection is - // already authenticated and enrolment is what happens over it. - if management, err := broker.ManagementFromEnvironment(); err == nil { - if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil { - return err - } - fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n", - issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange) - } else if !errors.Is(err, broker.ErrNotConfigured) { - return err - } - - made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret} - - // Absent is a state, not a failure: a control plane can hold records and a key before it has - // a broker. What it cannot do is issue a token anybody could use, and Missing() says so. - known, err := broker.FromEnvironment() - switch { - case err == nil: - made.Broker, made.Fingerprint = known.Address, known.Fingerprint - case errors.Is(err, broker.ErrNotConfigured): - default: - return err - } - encoded, err := made.Encode() - if err != nil { - return err - } - - fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n", - issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded) - fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.") - - if missing := made.Missing(); len(missing) > 0 { - fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n") - for _, m := range missing { - fmt.Printf(" - %s\n", m) - } - fmt.Printf("\nSet %s and %s once the broker is raised.\n", - broker.AddressVar, broker.CertificateVar) - } - return nil -} - -func openIdentity(ctx context.Context) (*identity.Identity, error) { - ident, err := identity.Open(ctx) - if err != nil { - return nil, err - } - if err := ident.Ready(ctx, 30*time.Second); err != nil { - ident.Close() - return nil, err - } - return ident, nil -} - -func identityCommand(ctx context.Context, args []string) error { - if len(args) == 0 || args[0] != "show" { - return errors.New("identity show") - } - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // Establish rather than read: a control plane asked for its identity before it has one should - // get one, not an error. Generating it is idempotent, so this is safe to run at any time. - key, err := ident.Establish(ctx) - if err != nil { - return err - } - fmt.Printf("signing key %s\n", key.ID) - fmt.Printf("fingerprint %s\n", key.Fingerprint()) - fmt.Printf("created %s\n", key.Created.Format(time.RFC3339)) - fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" + - "declaration because it carries a signature this key made (novox/hq ADR 0004).\n") - return nil -} - -func brokerCommand(args []string) error { - if len(args) == 0 || args[0] != "show" { - return errors.New("broker show") - } - known, err := broker.FromEnvironment() - if errors.Is(err, broker.ErrNotConfigured) { - fmt.Printf("no broker configured. Set %s and %s.\n\n"+ - "Until then tokens carry the signing key and the one-time secret, and say what they\n"+ - "are missing. They cannot be used to join.\n", - broker.AddressVar, broker.CertificateVar) - return nil - } - if err != nil { - return err - } - fmt.Printf("address %s\n", known.Address) - fmt.Printf("fingerprint %s\n", known.Fingerprint) - fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" + - "node checks it before sending anything (novox/hq ADR 0004).\n") - return nil -} - -// serve is the control plane running: one connection to the broker, one queue, one consumer. -func serve(ctx context.Context) error { - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // Established at start rather than on first use. A control plane that cannot sign is one - // whose declarations every node correctly refuses, and that should be a startup failure - // rather than something discovered at the first declaration. - key, err := ident.Establish(ctx) - if err != nil { - return err - } - fmt.Printf("signing as %s\n", key.Fingerprint()[:16]) - - management, err := broker.ManagementFromEnvironment() - if err != nil && !errors.Is(err, broker.ErrNotConfigured) { - return err - } - - // Where the broker is and what to expect there, so a node can be told how to come back - // without a person and a new token. - known, err := broker.FromEnvironment() - if err != nil && !errors.Is(err, broker.ErrNotConfigured) { - return err - } - if errors.Is(err, broker.ErrNotConfigured) { - fmt.Printf("no broker address configured, so enrolled nodes will not be told how to "+ - "reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar) - } - - work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known} - server, err := link.Connect(work, work) - if err != nil { - return err - } - defer server.Close() - // And build results nobody was waiting for. A build triggered any other way than `build` - // would otherwise be reported into the void, which is the same as not reporting it. - server.Records(builds{inv}) - - return server.Serve(ctx) -} - -// declare sends one node a declaration, signed. -// -// Signed here rather than trusted from the broker: a node connects to the broker and takes -// instruction from the control plane behind it, and those are two identities. If a node believed -// whatever arrived on its queue, a compromised broker could forge declarations — and since the -// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004). -func declare(ctx context.Context, args []string) error { - if len(args) != 2 { - return errors.New("declare ") - } - node, path := args[0], args[1] - - raw, err := os.ReadFile(path) - if err != nil { - return err - } - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // The node has to exist before it can be told anything. Publishing to a queue nobody consumes - // would sit there looking like success. - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - if _, err := inv.NodeByName(ctx, node); err != nil { - return err - } - - server, err := link.Connect(nil, nil) - if err != nil { - return err - } - defer server.Close() - - if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil { - return err - } - fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) - return nil -} - -// OverlayCIDRVar is the range the mesh allocates node addresses from. -const OverlayCIDRVar = "MESH_OVERLAY_CIDR" - -func overlayCIDR() string { - if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" { - return v - } - return "10.42.0.0/16" -} - -func overlayCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("overlay place [flags], or overlay show") - } - // Answered before anything is opened. A message about which command to use should not need a - // database to say so, and needing one turns a redirect into a connection error. - if args[0] == "push" { - return errors.New("`overlay push` is now `push`, which sends a node its network AND " + - "what its assignments resolve to — the two are computed from one picture of the " + - "mesh, and sending them separately would let them disagree") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - switch args[0] { - case "place": - return overlayPlace(ctx, inv, args[1:]) - case "show": - return overlayShow(ctx, inv) - - default: - return fmt.Errorf("overlay has no %q; it has place and show", args[0]) - } -} - -func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error { - if len(args) == 0 { - return errors.New("overlay place [--endpoint host:port] [--site name] [--hub]") - } - node := args[0] - - set := flag.NewFlagSet("overlay place", flag.ContinueOnError) - endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere") - site := set.String("site", "", "where this machine physically is, or empty if it roams") - hub := set.Bool("hub", false, "this node is the hub every other routes through") - if err := set.Parse(args[1:]); err != nil { - return err - } - - // Declared, all three. The address is evidence of reachability and is not the fact, and hub - // election by address prefix fails silently (novox/hq ADR 0007). - if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil { - return err - } - found, err := inv.NodeByName(ctx, node) - if err != nil { - return err - } - address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR()) - if err != nil { - return err - } - - fmt.Printf("%s is at %s on the overlay\n", node, address) - switch { - case *hub: - fmt.Println(" the hub — every node not sharing a site routes through it") - case *endpoint == "": - fmt.Println(" not dialable — it opens every path itself") - } - if *site != "" { - fmt.Printf(" at %s, so it peers directly with anything else there\n", *site) - } - return nil -} - -// network builds the private network over the machines that resolved the module for it. -// -// Not over every node the mesh knows. **A machine is on the private network because it was given -// the module**, and one that was not is absent from every peer list and from the names — which is -// the only thing "not on the network" can mean. Until this, having an address was enough, and -// there was no way to keep a machine off. -// -// Every node at once, which is the whole reason this is the control plane's work: a peer list is -// derived from all the others, so no node could compute its own. -func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, - refused map[string]string) (*overlay.Generator, error) { - places, err := inv.Overlays(ctx) - if err != nil { - return nil, err - } - nodes := make([]overlay.Node, 0, len(places)) - for _, p := range places { - if !on[p.Name] { - continue - } - nodes = append(nodes, overlay.Node{ - Name: p.Name, Key: p.Key, Endpoint: p.Endpoint, - Site: p.Site, Hub: p.Hub, Address: p.Address, - }) - } - if len(nodes) == 0 { - // Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this - // answers rather than refusing -- Compute would refuse for want of a hub, and reporting - // "no hub" to somebody who never asked for a network would be a lie about the cause. - return overlay.Empty(), nil - } - g, err := overlay.From(nodes, overlayCIDR(), "") - if err != nil && len(refused) > 0 { - // The network is missing something, and some machines could not be resolved at all. Those - // are almost always the same fact: a node that does not resolve contributes nothing, so - // reporting "no hub" would name a consequence and hide the cause. - var who []string - for name, why := range refused { - who = append(who, fmt.Sprintf(" %s: %s", name, why)) - } - sort.Strings(who) - return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+ - "probably why:\n%s", err, len(refused), strings.Join(who, "\n")) - } - return g, err -} - -// graph is the whole mesh's network, for showing it. -func graph(ctx context.Context, inv *inventory.Inventory) ([]overlay.Node, overlay.Graph, error) { - on, refused, err := whoResolves(ctx, inv, overlay.Requirement) - if err != nil { - return nil, nil, err - } - g, err := network(ctx, inv, on, refused) - if err != nil { - return nil, nil, err - } - return g.Nodes(), g.Graph(), nil -} - -// whoResolves is the machines whose resolution answers a requirement, and why the others did not. -// -// By what a module **provides**, not by its name. WireGuard is one way to have a private network -// and there could be others, so a machine is on the network because something it runs provides -// one — asking for a particular module by name would be the mistake this whole mechanism exists -// to avoid. -// -// Resolved rather than read from the assignment table, because a module can arrive by being -// required by something else, and a machine that needs the private network to do its job is on it -// for the same reason as one that was handed it directly. -func whoResolves(ctx context.Context, inv *inventory.Inventory, requirement string) ( - map[string]bool, map[string]string, error) { - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, nil, err - } - on := map[string]bool{} - // Why a node could not be resolved, kept rather than raised: one broken node must not stop - // the rest being described, and whoever is rendering that node will raise it themselves. - refused := map[string]string{} - for _, n := range nodes { - plan, _, err := planFor(ctx, inv, n.Name) - if err != nil { - refused[n.Name] = err.Error() - continue - } - for _, m := range plan.Modules { - for _, offered := range m.Offers() { - if offered == requirement { - on[n.Name] = true - } - } - } - } - return on, refused, nil -} - -// rendering is everything a declaration needs, computed over the whole mesh. -func generators(ctx context.Context, inv *inventory.Inventory) ( - map[string]catalogue.Generator, error) { - on, refused, err := whoResolves(ctx, inv, overlay.Addressing) - if err != nil { - return nil, err - } - net, err := network(ctx, inv, on, refused) - if err != nil { - return nil, err - } - // Both generators see the same machines: the ones on the private network. Names for a machine - // that is not on it would resolve to addresses it cannot reach, which is worse than no names. - return map[string]catalogue.Generator{ - overlay.Name: net, - overlay.Names: overlay.NamesFor(net.Nodes()), - overlay.Resolver: overlay.ResolverFor(net.Nodes()), - }, nil -} - -func overlayShow(ctx context.Context, inv *inventory.Inventory) error { - nodes, computed, err := graph(ctx, inv) - if err != nil { - return err - } - if len(nodes) == 0 { - // Not "this mesh has no nodes", which it said until the network became a module and was - // then a lie about the cause: a mesh can have every node it will ever have and nobody on - // the private network, because nobody asked for one. - fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n", - overlay.Name) - return nil - } - - for _, n := range nodes { - place := n.Address - if place == "" { - // Said, not skipped. A node with no place is a node with no network, and it should - // be visible here rather than quietly absent from a list of who is on it. - place = "no address — run `overlay place`" - } - fmt.Printf("%-16s %-14s", n.Name, place) - switch { - case n.Hub: - fmt.Print(" hub") - case !n.Reachable(): - fmt.Print(" not dialable") - } - if n.Site != "" { - fmt.Printf(" at %s", n.Site) - } - fmt.Println() - for _, p := range computed[n.Name] { - fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why) - } - } - return nil -} - -// SilentFor is how long a node may be quiet before the mesh says so. -// -// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number -// is not the point — being able to say "out of touch" at all is, and nothing could before. -const SilentFor = 3 * time.Minute - -// heardFrom says when a node was last heard from, in a form somebody can act on. -// -// "never" and "an hour ago" are different answers and are kept different. A node that has never -// spoken did not finish joining; a node last heard from an hour ago is running an hour-old -// picture of the mesh. -func heardFrom(n inventory.Node) string { - silent, ever := n.Silent() - switch { - case !ever: - return "never spoken" - case silent > SilentFor: - return "out of touch " + roughly(silent) - default: - return "here" - } -} - -// roughly is a duration a person reads rather than parses. -func roughly(d time.Duration) string { - switch { - case d < time.Hour: - return fmt.Sprintf("%dm", int(d.Minutes())) - case d < 48*time.Hour: - return fmt.Sprintf("%dh", int(d.Hours())) - default: - return fmt.Sprintf("%dd", int(d.Hours()/24)) - } -} - -func moduleCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("module add , module list, or module forget ") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - switch args[0] { - case "add": - set := flag.NewFlagSet("module add", flag.ContinueOnError) - repo := set.String("source", "", "where this module comes from") - ref := set.String("ref", "", "the branch followed there") - commit := set.String("commit", "", "the commit this manifest was read at") - positionals, err := parseAround(set, args[1:]) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("module add [--source --ref --commit ]") - } - raw, err := os.ReadFile(positionals[0]) - if err != nil { - return err - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - return err - } - // Provenance together or not at all. A source with no commit cannot be compared against - // anything, so it would record where the module came from and still never be able to say - // the mesh is behind it — which is the one thing recording it is for. - if (*repo == "") != (*commit == "") { - return errors.New("--source and --commit go together: a source with no commit " + - "cannot be compared against anything, and a commit with no source has nothing " + - "to be compared with") - } - if err := inv.RegisterModule(ctx, m, inventory.Source{ - Repository: *repo, Ref: *ref, BuiltFrom: *commit, - }); err != nil { - return err - } - fmt.Printf("%s registered", m.Module) - if *commit != "" { - fmt.Printf(" from %s", short(*commit)) - } - if len(m.Provides) > 0 { - fmt.Printf(", providing %s", describeOffers(m.Provides)) - } - fmt.Println() - for _, c := range m.Claims { - fmt.Printf(" claims %s, one per %s\n", c.Name, c.At()) - } - return nil - - case "list": - // The catalogue: what exists, where it came from, whether it is current, and who runs it. - // The provenance was recorded from the first build and nothing showed it, which made - // "is this current?" a question you could only answer by reading the database. - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - if len(entries) == 0 { - fmt.Println("this mesh knows about no modules yet") - return nil - } - var stale int - for _, e := range entries { - m := e.Manifest - fmt.Printf("%-18s %-8s", m.Module, m.Version) - - switch { - case e.Provided: - fmt.Printf(" %-22s", "with the control plane") - case e.Source.Repository == "": - // Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and - // worth saying, because nothing can rebuild it. - fmt.Printf(" %-22s", "handed over") - case !e.Source.Current(): - stale++ - fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head)) - default: - fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom)) - } - - if len(e.On) > 0 { - fmt.Printf(" on %s", strings.Join(e.On, ", ")) - } else { - fmt.Printf(" on nothing") - } - fmt.Println() - - var says []string - if len(m.Provides) > 0 { - says = append(says, "provides "+describeOffers(m.Provides)) - } - if len(m.Requires) > 0 { - says = append(says, "requires "+strings.Join(m.Requires, ", ")) - } - for _, c := range m.Claims { - says = append(says, "claims "+c.At()+"/"+c.Name) - } - if len(m.Capabilities) > 0 { - says = append(says, "needs "+strings.Join(m.Capabilities, ", ")) - } - if len(says) > 0 { - fmt.Printf(" %s\n", strings.Join(says, " · ")) - } - } - if stale > 0 { - fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale) - } - return nil - - case "moved": - if len(args) != 3 { - return errors.New("module moved — the source has a newer commit") - } - if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil { - return err - } - from, err := inv.SourceOf(ctx, args[1]) - if err != nil { - return err - } - if from.Current() { - fmt.Printf("%s is current at %s\n", args[1], short(from.Head)) - return nil - } - fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n", - args[1], short(from.BuiltFrom), short(from.Head)) - fmt.Printf(" run `build %s` to catch up\n", from.Repository) - return nil - - case "forget": - if len(args) != 2 { - return errors.New("module forget ") - } - if err := inv.ForgetModule(ctx, args[1]); err != nil { - return err - } - fmt.Printf("%s forgotten\n", args[1]) - return nil - - default: - return fmt.Errorf("module has no %q; it has add, list, moved and forget", args[0]) - } -} - -func assignCommand(ctx context.Context, verb string, args []string) error { - if len(args) != 2 { - return fmt.Errorf("%s ", verb) - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - if verb == "unassign" { - if err := inv.Unassign(ctx, args[0], args[1]); err != nil { - return err - } - fmt.Printf("%s no longer runs %s — run `push %s` to make it so\n", args[0], args[1], args[0]) - return nil - } - if err := inv.Assign(ctx, args[0], args[1]); err != nil { - return err - } - fmt.Printf("%s is assigned %s\n", args[0], args[1]) - - // Resolved immediately, because an assignment that cannot be applied should be said now - // rather than at the next push. The assignment is kept either way: it is what a person meant, - // and the refusal is about the set rather than about this one. - if _, _, err := planFor(ctx, inv, args[0]); err != nil { - fmt.Println() - return err - } - fmt.Printf(" run `push %s` to send it\n", args[0]) - return nil -} - -// planFor works out everything a node should run, from what was assigned to it. -func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) { - shelf, err := inv.Catalogue(ctx) - if err != nil { - return catalogue.Resolution{}, nil, err - } - assigned, err := inv.Assigned(ctx, nodeName) - if err != nil { - return catalogue.Resolution{}, nil, err - } - capabilities, err := inv.ProfileOf(ctx, nodeName) - if err != nil { - return catalogue.Resolution{}, nil, err - } - - places, err := inv.Overlays(ctx) - if err != nil { - return catalogue.Resolution{}, nil, err - } - var site string - for _, p := range places { - if p.Name == nodeName { - site = p.Site - } - } - - world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName) - if err != nil { - return catalogue.Resolution{}, nil, err - } - world.Pinned, err = inv.PinsFor(ctx, nodeName) - if err != nil { - return catalogue.Resolution{}, nil, err - } - - onNetwork, err := whereEveryoneIs(ctx, inv, shelf) - if err != nil { - return catalogue.Resolution{}, nil, err - } - - // What this mesh can answer with a record rather than a machine, and which record each of - // this node's modules was put on. Read across a context boundary by name, which is what - // crossing one is allowed to carry (novox/hq ADR 0008). - world.Licences, world.Using, err = licencesFor(ctx, nodeName) - if err != nil { - return catalogue.Resolution{}, nil, err - } - - resolved, err := catalogue.Resolve(shelf, assigned, - catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities, - At: onNetwork[nodeName]}, world) - if err != nil { - return catalogue.Resolution{}, nil, err - } - - // The credential for each thing this node takes from elsewhere. Made once and kept, so the - // password a provider is told to create is the one its consumer was given — and sealed to - // this node before it was ever written down, so nothing between here and there can read it. - for i, n := range resolved.Needs { - if n.ByRecord { - // Answered by something the mesh holds, so there is no pair-wise secret between two - // machines. Its key was supplied by a person and sealed to this node then; the mesh - // discarded the plaintext and cannot make another. - sealed, err := keyFor(ctx, n.From, nodeName, n.For) - if err != nil { - return catalogue.Resolution{}, nil, err - } - resolved.Needs[i].Sealed = sealed - continue - } - secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.From) - if err != nil { - // Said rather than skipped. A machine that resolves cleanly and receives no - // credential is one that will fail to authenticate at some later, less obvious - // moment. - return catalogue.Resolution{}, nil, fmt.Errorf( - "%s needs %s from %s and no credential could be made for it: %w", - nodeName, n.Name, n.From, err) - } - resolved.Needs[i].Sealed = secret.ForConsumer - } - - // Settings for everything that resolved, including modules nobody assigned directly: a - // requirement pulled in by something else is still configurable, and finding out that it is - // not only when you try would be an arbitrary line nobody could predict. - settings := catalogue.SettingsBy{} - var stray []string - for _, m := range resolved.Modules { - layers, err := inv.SettingsFor(ctx, nodeName, m.Module) - if err != nil { - return catalogue.Resolution{}, nil, err - } - if len(layers) == 0 { - continue - } - settings[m.Module] = layers - stray = append(stray, catalogue.UnusedSettings(m, layers)...) - } - if len(stray) > 0 { - // Somebody set something that reaches no file. Said here rather than discovered by the - // machine not behaving differently, which is the slowest way there is. - return catalogue.Resolution{}, nil, fmt.Errorf( - "these settings reach nothing:\n - %s", strings.Join(stray, "\n - ")) - } - return resolved, settings, nil -} - -// theRestOfTheMesh is what every other node holds and offers. -// -// Two things at once because they come from the same place — resolving the other nodes — and -// because both are facts about what is actually running rather than records that could disagree -// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node -// runs; neither is a table somebody keeps up to date. -// -// **Two passes over the others.** What a node offers the mesh needs that node resolved, and -// resolving it may need what the mesh offers. So the first pass takes brokered requirements on -// trust and answers only *what does each node offer*; the second answers everything with that in -// hand. Nothing is ever declared from the first. -func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, - shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) { - - // Every node, not only the placed ones. A machine that was never put on the private network - // still runs modules, still holds claims, and still offers whatever it offers. - nodes, err := inv.Nodes(ctx) - if err != nil { - return catalogue.World{}, err - } - places, err := inv.Overlays(ctx) - if err != nil { - return catalogue.World{}, err - } - siteOf := map[string]string{} - for _, p := range places { - siteOf[p.Name] = p.Site - } - // Which machines are actually on the private network, and what they are called there. Not - // "has an address" — that was true of every placed machine and told you nothing about whether - // anything could reach it. It is what resolved the module. - onNetwork, err := whereEveryoneIs(ctx, inv, shelf) - if err != nil { - return catalogue.World{}, err - } - - type candidate struct { - node catalogue.Node - assigned []string - } - var others []candidate - for _, n := range nodes { - if n.Name == exclude { - continue - } - theirs, err := inv.Assigned(ctx, n.Name) - if err != nil || len(theirs) == 0 { - continue - } - caps, _ := inv.ProfileOf(ctx, n.Name) - others = append(others, candidate{ - catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps, - At: onNetwork[n.Name]}, theirs}) - } - - offered := map[string][]catalogue.Provider{} - for _, o := range others { - got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true}) - if err != nil { - // Their set does not resolve for some other reason. Not this node's problem to - // report, and nothing of theirs is running, so it offers nothing. - continue - } - for _, m := range got.Modules { - for _, name := range m.OffersAt(catalogue.ScopeMesh) { - // What that module says a consumer needs to know, with that node's settings on - // it: a port somebody moved on the provider is a port its consumers must be told - // about, and the two coming from different places is how they come to disagree. - serves := m.Serves[name] - if len(serves) > 0 { - layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) - if err != nil { - return catalogue.World{}, err - } - serves, err = catalogue.Settle(serves, layers) - if err != nil { - return catalogue.World{}, err - } - } - offered[name] = append(offered[name], catalogue.Provider{ - Node: o.node.Name, At: o.node.At, Serves: serves}) - } - } - } - for k := range offered { - sort.Slice(offered[k], func(i, j int) bool { - return offered[k][i].Node < offered[k][j].Node - }) - } - - world := catalogue.World{Offered: offered} - for _, o := range others { - got, err := catalogue.Resolve(shelf, o.assigned, o.node, world) - if err != nil { - continue - } - world.Held = append(world.Held, got.Claims...) - } - return world, nil -} - -// whereEveryoneIs is each machine's name on the private network, for the ones on it. -// -// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every -// other path here answers a question about one node by resolving the others; this one is called -// *from* that path, so doing the same would not terminate — which it did not, for two minutes, -// until it was run. -// -// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the -// private network depends on what it was assigned and what that requires, both of which are local -// facts. What it takes *from* other machines does not change the answer. -// -// The distinction that matters is kept: a machine absent from the network module's own view is -// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the -// network became something a machine is given. -func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory, - shelf map[string]catalogue.Manifest) (map[string]string, error) { - - if shelf == nil { - // Refused rather than answered. Being on the private network is a conclusion about what a - // node resolves to, so with no catalogue nothing resolves and the honest answer is - // "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused - // every certificate the mesh was asked for while saying the machine was on no network. - return nil, errors.New( - "asked where everyone is without the catalogue, which cannot be answered") - } - places, err := inv.Overlays(ctx) - if err != nil { - return nil, err - } - out := map[string]string{} - for _, p := range places { - if p.Address == "" { - continue - } - assigned, err := inv.Assigned(ctx, p.Name) - if err != nil || len(assigned) == 0 { - continue - } - caps, _ := inv.ProfileOf(ctx, p.Name) - got, err := catalogue.Resolve(shelf, assigned, - catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps}, - catalogue.World{Unchecked: true}) - if err != nil { - continue - } - for _, m := range got.Modules { - for _, offered := range m.Offers() { - if offered == overlay.Requirement { - out[p.Name] = overlay.InternalName(p.Name) - } - } - } - } - return out, nil -} - -// declarationFor is everything a node would be sent. -// -// One place, because there were three and one of them was written before credentials existed and -// silently produced a declaration missing them — a difference between what `plan` showed and what -// `plan --json` handed to anything reading it. -func declarationFor(ctx context.Context, inv *inventory.Inventory, node string, - plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) { - gens, err := generators(ctx, inv) - if err != nil { - return nil, err - } - return declarationWith(ctx, inv, node, plan, settings, gens) -} - -// declarationWith is the same, for a caller that has already worked out the generators once and -// is about to use them for every node. -func declarationWith(ctx context.Context, inv *inventory.Inventory, node string, - plan catalogue.Resolution, settings catalogue.SettingsBy, - gens map[string]catalogue.Generator) ([]map[string]any, error) { - grants, err := grantsFor(ctx, inv, node) - if err != nil { - return nil, err - } - // And each module's own secrets — a superuser password, an administrator, an account. Made - // per node, so a module running on three machines has three. - needed := map[string]map[string]string{} - for _, m := range plan.Modules { - for name := range m.Needs { - sealed, err := inv.SecretForModule(ctx, node, m.Module, name) - if err != nil { - return nil, err - } - if needed[m.Module] == nil { - needed[m.Module] = map[string]string{} - } - needed[m.Module][name] = sealed - } - } - // And a certificate for this machine's name inside the mesh, when anything on it asks. Issued - // rather than stored: the node's key does not change, so signing again produces an equally - // valid certificate and there is nothing to keep in step. - var certificate, authority string - for _, m := range plan.Modules { - if m.Certificate == nil { - continue - } - issued, meshCA, err := certificateFor(ctx, inv, node) - if err != nil { - return nil, err - } - certificate, authority = issued, meshCA - break - } - - // And who else is on the private network, which is what a rule saying "from the mesh" - // resolves to. Every node's address, including this one's: a machine reaching itself by its - // own overlay address rather than by loopback is ordinary, and leaving it out would filter - // the node's own traffic to itself with no rule naming why. - private, err := onThePrivateNetwork(ctx, inv) - if err != nil { - return nil, err - } - - // And every machine's name, so a container can reach one. The same set that writes the - // machine's own hosts file — one reading, so a container and its machine cannot disagree - // about where another machine is. - names, err := namesInTheMesh(ctx, inv) - if err != nil { - return nil, err - } - - return plan.Declaration(catalogue.Rendering{ - Settings: settings, Generators: gens, Grants: grants, Needed: needed, - Certificate: certificate, Authority: authority, Mesh: private, Names: names}) -} - -// onThePrivateNetwork is every node's address on the overlay, sorted. -// -// A node with no address is left out rather than rendered as an empty source: an empty entry in a -// source set is a syntax error in the rule file, and a rule file that does not load leaves the -// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule, -// because nothing reports it. -func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory) ([]string, error) { - places, err := inv.Overlays(ctx) - if err != nil { - return nil, err - } - var out []string - for _, p := range places { - if strings.TrimSpace(p.Address) != "" { - out = append(out, p.Address) - } - } - sort.Strings(out) - return out, nil -} - -// certificateFor is what the mesh certifies about one machine's internal name. -// -// It reaches across two contexts and reads neither one's store from the other: `inventory` knows -// the machine and whether it is on the private network, `identity` holds the authority and the -// key that machine reported. The process holding both grants asks each for its part -// (novox/hq ADR 0008). -func certificateFor(ctx context.Context, inv *inventory.Inventory, node string) (string, string, error) { - ident, err := openIdentity(ctx) - if err != nil { - return "", "", err - } - defer ident.Close() - - record, err := inv.NodeByName(ctx, node) - if err != nil { - return "", "", err - } - serving, err := ident.ServingKeyOf(ctx, record.ID) - if err != nil { - return "", "", err - } - if serving == "" { - // The machine joined before it had one, or never reported it. Said plainly, because the - // remedy is on the machine and no amount of pushing from here will produce one. - return "", "", fmt.Errorf( - "%s wants a certificate and has never told the mesh what key it serves with; it "+ - "joins again to report one", node) - } - - // The name it is certified for. Only a machine on the private network has one — a certificate - // for a name nothing resolves is a certificate nothing can check. - // - // With the catalogue, not without it. Being on the private network is a conclusion about what - // a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no - // network — which refused every certificate the mesh was asked for, and said the machine was - // not on a network it plainly was. - shelf, err := inv.Catalogue(ctx) - if err != nil { - return "", "", err - } - where, err := whereEveryoneIs(ctx, inv, shelf) - if err != nil { - return "", "", err - } - name := where[node] - if name == "" { - return "", "", fmt.Errorf( - "%s wants a certificate and is not on the private network, so it has no name inside "+ - "the mesh to be certified for", node) - } - - issued, err := ident.Certify(ctx, node, name, serving) - if err != nil { - return "", "", err - } - authority, err := ident.EstablishAuthority(ctx) - if err != nil { - return "", "", err - } - return issued, authority.Certificate, nil -} - -// grantsFor is every credential this node must create, because something elsewhere uses it. -// -// The mirror of what a consumer is given, and the half that makes the credential real: a password -// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so -// the mesh hands over something it cannot itself use. -func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.Grant, error) { - issued, err := inv.SecretsFrom(ctx, node) - if err != nil { - return nil, err - } - - // Where each consumer is, so a provider that must reach back to one does not have to know how - // the mesh names machines. - shelf, err := inv.Catalogue(ctx) - if err != nil { - return nil, err - } - onNetwork, err := whereEveryoneIs(ctx, inv, shelf) - if err != nil { - return nil, err - } - - // What each consumer actually asked for, taken from that machine's own resolution rather than - // from a record beside it. A provider told to create a password and not what to create it for - // can do nothing with it, and the name a consumer wants is the consumer's to say. - out := make([]catalogue.Grant, 0, len(issued)) - for _, s := range issued { - plan, settings, err := planFor(ctx, inv, s.Consumer) - if err != nil { - // Their set does not resolve. Skipped rather than fatal: this node is not the place - // to report another machine's problem, and a grant for something that is not going to - // run would have the provider create a user nothing uses. - continue - } - from, values, err := plan.ContributionsTo(s.Name, settings) - if err != nil { - return nil, err - } - out = append(out, catalogue.Grant{ - Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer], - From: from, Values: values, Sealed: s.ForProvider}) - } - return out, nil -} - -func planCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("plan", flag.ContinueOnError) - // Because "one resource" does not tell you whether the settings landed. Being able to read - // the file before it is sent is the difference between believing a merge worked and knowing. - show := set.Bool("files", false, "print the files this node would be given") - // The declaration exactly as the node would receive it. For handing to something else -- - // checking it against the host's own parser, most usefully, which is the only way to know - // that what the control plane emits is what the host accepts. - asJSON := set.Bool("json", false, "print the declaration this node would be sent") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("plan [--files] [--json]") - } - args = positionals - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - plan, settings, err := planFor(ctx, inv, args[0]) - if err != nil { - return err - } - if len(plan.Modules) == 0 { - fmt.Printf("%s is assigned nothing\n", args[0]) - return nil - } - if *asJSON { - resources, err := declarationFor(ctx, inv, args[0], plan, settings) - if err != nil { - return err - } - body, err := json.MarshalIndent( - map[string]any{"declaration": 1, "resources": resources}, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - - fmt.Printf("%s would run:\n", args[0]) - for _, m := range plan.Modules { - fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module]) - } - for _, c := range plan.Claims { - fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope) - } - // What this machine depends on that is not on it. Worth saying out loud: it is the only part - // of a node's set that stops working when a *different* machine goes away, and nothing else - // in this output would have told anybody that. - for _, n := range plan.Needs { - fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) - } - resources, err := declarationFor(ctx, inv, args[0], plan, settings) - if err != nil { - return err - } - for module, layers := range settings { - for _, layer := range layers { - fmt.Printf(" %-20s settings from %s\n", module, layer.From) - } - } - fmt.Printf("\n%d resource(s)\n", len(resources)) - - if *show { - for _, r := range resources { - content, ok := r["content"].(string) - if !ok { - continue - } - fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content) - } - } - return nil -} - -// pushCommand sends nodes everything they should be: their place on the network, and what their -// assignments resolve to. -// -// One declaration, not two. A node holding its network and not its modules, or the reverse, is -// half-configured for as long as that lasts — and the two are computed from the same picture of -// the mesh, so sending them apart would let them disagree. -func pushCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("push", flag.ContinueOnError) - // Only the machines that need it. - // - // **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is - // a scheduler over this, and building the scheduler first would mean two paths to one act - // with nothing to compare them against. A person can run this; so can cron; so can whatever - // eventually watches. - behind := set.Bool("behind", false, - "only machines whose last declaration was refused or partly failed") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - args = positionals - if len(args) > 1 { - return errors.New("push [] [--behind] — one node, or all of them") - } - if len(args) == 1 && *behind { - // Naming a machine and asking for the ones that need it are two different requests, and - // guessing which was meant would sometimes push to a machine somebody did not name. - return errors.New("push or push --behind, not both: one names a machine and the " + - "other asks which machines need one") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // Every node, not only the ones on the private network. A machine that was never given the - // network module still takes modules, and iterating the network here is what used to make - // "on the network" and "managed" the same thing. - nodes, err := inv.Nodes(ctx) - if err != nil { - return err - } - - // Which machines are not in the state they were sent, when that is what was asked for. - var needsOne map[string]inventory.Doing - if *behind { - wrong, err := inv.NotDoingWhatTheyWereTold(ctx) - if err != nil { - return err - } - needsOne = map[string]inventory.Doing{} - for _, d := range wrong { - needsOne[d.Node] = d - } - // **And every machine not running what the mesh would send it.** "Behind" used to mean - // only "failed or refused", so a machine that applied cleanly and whose declaration has - // since changed was not behind — and novox/hq ADR 0010's question, *did my change go - // out?*, was answerable only for the machines that broke. - would, err := wouldSend(ctx, inv, nodes) - if err != nil { - return err - } - waiting, err := inv.Waiting(ctx, would) - if err != nil { - return err - } - for _, m := range waiting { - if _, already := needsOne[m.Node]; already { - continue - } - needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} - } - if len(needsOne) == 0 { - // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" - // must never look the same. - fmt.Println("every machine is doing what it was told") - return nil - } - } - gens, err := generators(ctx, inv) - if err != nil { - return err - } - - server, err := link.Connect(nil, nil) - if err != nil { - return err - } - defer server.Close() - - // Every node is resolved before anything is sent. A push that configured three nodes and then - // refused on the fourth would leave the mesh in a state nobody asked for, and the fourth is - // exactly where a claim collision shows up. - type ready struct { - node string - resources []map[string]any - } - var sending []ready - var refusals []string - - for _, n := range nodes { - if len(args) == 1 && n.Name != args[0] { - continue - } - if *behind { - doing, needs := needsOne[n.Name] - if !needs { - continue - } - // A machine that has been failing the same way for a long time is not going to stop - // because it was asked again. Said, and pushed to anyway — refusing would leave no - // way to retry after fixing the cause, and this is a command somebody ran. - // - // Only for machines that reported something. One that is merely waiting has no report - // to be old, and saying it had been failing since the zero time would be a sentence - // about nothing. - if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { - fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ - "unlikely to be timing\n", - n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) - } - } - plan, settings, err := planFor(ctx, inv, n.Name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err)) - continue - } - // The private network is in here with everything else. It used to be composed separately - // and prepended, which meant every machine with an address was on it and no machine could - // be kept off. It is a module now, so it arrives the way a module does. - resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err)) - continue - } - if len(resources) == 0 { - fmt.Printf("%s is assigned nothing — skipped\n", n.Name) - continue - } - sending = append(sending, ready{n.Name, resources}) - } - - if len(refusals) > 0 { - return fmt.Errorf("nothing was sent. %d node(s) could not be resolved:\n\n%s", - len(refusals), strings.Join(refusals, "\n\n")) - } - - for _, s := range sending { - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) - if err != nil { - return err - } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { - return err - } - // After it is away, not before. A digest recorded for something that failed to send would - // make the machine look current for a declaration it never received. - record, err := inv.NodeByName(ctx, s.node) - if err != nil { - return err - } - if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { - return err - } - fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) - } - fmt.Printf("\n%d node(s) told\n", len(sending)) - return nil -} - -// sendTo resolves and sends to exactly the machines named, or refuses without sending anything. -// -// The same all-or-nothing rule push follows, and for the same reason: a rotation that reached the -// consumer and refused on the provider would leave one end holding a credential the other has -// never heard of — which is the state this whole mechanism exists to make impossible. -func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error { - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - gens, err := generators(ctx, inv) - if err != nil { - return err - } - - type ready struct { - node string - resources []map[string]any - } - var sending []ready - var refusals []string - for _, name := range names { - plan, settings, err := planFor(ctx, inv, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - resources, err := declarationWith(ctx, inv, name, plan, settings, gens) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - sending = append(sending, ready{name, resources}) - } - if len(refusals) > 0 { - return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", - len(refusals), strings.Join(refusals, "\n\n")) - } - - server, err := link.Connect(nil, nil) - if err != nil { - return err - } - defer server.Close() - - for _, s := range sending { - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) - if err != nil { - return err - } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { - return err - } - record, err := inv.NodeByName(ctx, s.node) - if err != nil { - return err - } - if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { - return err - } - fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) - } - return nil -} - -// short is a commit as a person refers to it. -func short(commit string) string { - if len(commit) > 8 { - return commit[:8] - } - return commit -} - -// statusCommand answers "did my change go out?". -// -// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a -// comparison: it is answerable today by opening a pipeline, and something has to replace that or -// this is worse to live with whatever its other properties. -// -// The answer is not "a job succeeded". It is which modules the mesh has not built from what their -// source now has, and which machines are running the old one. -func statusCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("status", flag.ContinueOnError) - asJSON := set.Bool("json", false, "the same answers, for something other than a person") - if _, err := parseAround(set, args); err != nil { - return err - } - - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - asked, err := theThreeQuestions(ctx, inv) - if err != nil { - return err - } - wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet - behind, sources := asked.behind, asked.sources - - if *asJSON { - body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting) - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - - if len(wrong) > 0 { - fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong)) - for _, d := range wrong { - fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04")) - if d.Refused != "" { - // The host's own words. It says exactly what it could not accept, and nothing - // written here would say it better. - fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused)) - } - for _, f := range d.Failed { - fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error)) - } - } - fmt.Println() - } - - if len(quiet) > 0 { - var said []string - for _, n := range quiet { - said = append(said, n.Name+" ("+heardFrom(n)+")") - } - fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n", - len(quiet), strings.Join(said, "\n ")) - } - - if len(behind) > 0 { - var names []string - for m := range behind { - names = append(names, m) - } - sort.Strings(names) - - fmt.Printf("%d module(s) behind their source:\n\n", len(behind)) - for _, m := range names { - from := sources[m] - fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head)) - if on := behind[m]; len(on) > 0 { - // The part somebody actually wants. A module being out of date is a fact about - // the catalogue; machines running the old one is the thing with consequences. - fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", ")) - } else { - fmt.Printf(" %-18s assigned to nothing\n", "") - } - } - // The remedy, beside the problem. A status that says what is wrong and not what to do - // about it makes somebody go and find the command, and the command is the whole point of - // having noticed. - fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n") - fmt.Println() - } - - if len(asked.waiting) > 0 { - // The other half of "is anything out of date": a module behind its source says the - // catalogue is old, and this says a machine is — and only this one has somebody's change - // waiting inside it. - var told, never []string - for _, m := range asked.waiting { - if m.Never { - never = append(never, m.Node) - continue - } - told = append(told, m.Node) - } - if len(told) > 0 { - fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n", - len(told), strings.Join(told, ", ")) - } - if len(never) > 0 { - // Never told is not out of date. The remedy is the same push and the situation is - // not the same at all: nobody has ever asked this machine to be anything. - fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", - len(never), strings.Join(never, ", ")) - } - fmt.Printf("\n `push --behind` sends them\n\n") - } - - if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 { - // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, - // and getting here means every question was asked and answered. - fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+ - "the mesh would send them, and every module current with its source\n", len(nodes)) - } - return nil -} - // parseAround reads flags that may sit before, after or between positional arguments. // // The standard library stops at the first non-flag argument, so `module add thing.json --source x` @@ -1973,797 +183,6 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) { } } -func settingsCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("settings set [--node ], or settings clear [--node ]") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - set := flag.NewFlagSet("settings", flag.ContinueOnError) - node := set.String("node", "", "one machine, rather than the whole mesh") - positionals, err := parseAround(set, args[1:]) - if err != nil { - return err - } - - where := "the whole mesh" - if *node != "" { - where = *node - } - - switch args[0] { - case "set": - if len(positionals) != 2 { - return errors.New("settings set [--node ]") - } - raw, err := os.ReadFile(positionals[1]) - if err != nil { - return err - } - var values map[string]any - if err := json.Unmarshal(raw, &values); err != nil { - return fmt.Errorf("%s is not a settings file: %w", positionals[1], err) - } - if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil { - return err - } - - var keys []string - for k := range values { - keys = append(keys, k) - } - sort.Strings(keys) - fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", ")) - fmt.Println(" run `push` to send it") - return nil - - case "clear": - if len(positionals) != 1 { - return errors.New("settings clear [--node ]") - } - if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil { - return err - } - fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where) - return nil - - default: - return fmt.Errorf("settings has no %q; it has set and clear", args[0]) - } -} - -// describeOffers says what a module provides, and marks the ones answered from anywhere in the -// mesh — because "provides a database" and "provides a shell" are read the same way and mean -// entirely different things about where the answer has to be. -func describeOffers(offers []catalogue.Offer) string { - var out []string - for _, o := range offers { - if o.At() == catalogue.ScopeMesh { - out = append(out, o.Name+" (from anywhere in the mesh)") - continue - } - out = append(out, o.Name) - } - return strings.Join(out, ", ") -} - -// pinCommand says which node a machine gets a provision from. -// -// Needed only when more than one could answer, and recordable before that -- a mesh with one -// database should not change where an existing machine gets its data the day a second one -// arrives. -func pinCommand(ctx context.Context, args []string, setting bool) error { - if setting && len(args) != 3 { - return errors.New("pin ") - } - if !setting && len(args) != 2 { - return errors.New("unpin ") - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - if !setting { - if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil { - return err - } - fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1]) - return nil - } - if args[0] == args[2] { - // Allowed by nothing here, and worth saying rather than resolving into a confusing - // refusal later: a node providing something to itself is a node-scoped provision, and - // this field is for the other kind. - return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+ - "provides, which does not need saying", args[0], args[1]) - } - if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil { - return err - } - fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2]) - fmt.Printf(" run `push %s` to send it\n", args[0]) - return nil -} - -// buildCommand builds a module from its source and records what came out. -// -// **Run where there is a container runtime**, which is why it is a command rather than something -// the control plane does on its own: building needs to run things on a machine, and what the -// control plane may send a machine is bounded by the declaration language. This is the shape the -// builder module will take when it is given work over the broker; today a person runs it, and the -// mesh records the result the same way either way. -func buildCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("build", flag.ContinueOnError) - ref := set.String("ref", "", "the branch, tag or commit to build") - wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer") - dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing") - // Every module whose source has moved, rather than one named repository. - // - // **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh - // already knows which modules are behind their source, so making a person read that list and - // retype each repository is asking them to be the loop. Naming a repository and asking which - // ones need building are different requests, so they are not combined. - behind := set.Bool("behind", false, "every module the mesh holds older than its source has") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if *behind { - if len(positionals) != 0 { - return errors.New("build or build --behind, not both: one names a " + - "repository and the other asks which need building") - } - return buildBehind(ctx, *wait) - } - if len(positionals) != 1 { - return errors.New("build [--ref R] [--wait D] [--dry-run]") - } - - if *dryRun { - return buildAndShow(ctx, positionals[0], *ref, *wait) - } - return buildOne(ctx, positionals[0], *ref, *wait) -} - -// buildFrom turns what a builder said into what the mesh keeps. -func buildFrom(result link.BuildResult) inventory.Build { - kept := inventory.Build{ - ID: result.ID, Repository: result.Repository, Ref: result.Ref, - Commit: result.Commit, On: result.On, Failed: result.Failed, - } - for _, made := range result.Made { - kept.Made = append(kept.Made, inventory.Artifact{ - Name: made.Name, Kind: made.Kind, Reference: made.Reference, - }) - } - // The module name comes from the manifest, which only exists when the build got that far. - if len(result.Manifest) > 0 { - if m, err := catalogue.ParseManifest(result.Manifest); err == nil { - kept.Module = m.Module - } - } - return kept -} - -// buildsCommand says what has been built lately. -func buildsCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("builds", flag.ContinueOnError) - limit := set.Int("n", 20, "how many to show") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - module := "" - if len(positionals) == 1 { - module = positionals[0] - } else if len(positionals) > 1 { - return errors.New("builds [] [-n N]") - } - - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - builds, err := inv.Builds(ctx, module, *limit) - if err != nil { - return err - } - if len(builds) == 0 { - // Said rather than printed as nothing: an empty list and a failed read must never look - // the same, and getting here means the store answered. - if module != "" { - fmt.Printf("nothing has been built for %s\n", module) - return nil - } - fmt.Println("nothing has been built yet") - return nil - } - - for _, b := range builds { - what := b.Module - if what == "" { - // It failed before knowing what it was building, which is most of the interesting - // failures. The repository is what a person has to go and look at. - what = "?" - } - outcome := "built " + short(b.Commit) - if !b.Worked() { - outcome = "failed" - } - fmt.Printf("%-18s %-14s %-10s %s\n", - what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04")) - fmt.Printf(" %s", b.Repository) - if b.Ref != "" { - fmt.Printf(" at %s", b.Ref) - } - fmt.Println() - for _, made := range b.Made { - fmt.Printf(" %-10s %s\n", made.Kind, made.Reference) - } - if !b.Worked() { - // The builder's own first line. The whole failure is often a build log, and printing - // it here would bury every other row. - fmt.Printf(" %s\n", firstLine(b.Failed)) - } - } - return nil -} - -// firstLine is as much of a failure as belongs in a list. -func firstLine(s string) string { - if cut := strings.IndexByte(s, '\n'); cut >= 0 { - return strings.TrimSpace(s[:cut]) - } - return strings.TrimSpace(s) -} - -// builds keeps what a builder said, for the serving control plane. -// -// A type of its own rather than a method on the enrolment, because they are unrelated things -// arriving on one queue and an implementation of one should not have to say anything about the -// other. -type builds struct{ inv *inventory.Inventory } - func (b builds) Built(ctx context.Context, result link.BuildResult) error { return b.inv.RecordBuild(ctx, buildFrom(result)) } - -// builderCommand issues a build machine its own broker credential. -// -// **A build machine is not a node**, and giving it a node's account would let it read another -// machine's declarations. This is narrower and different: read the build queue, write the -// exchange and an asker's reply queue, and nothing else. -// -// Issued rather than assumed, because until this the builder used whatever credential it was -// handed — which in practice meant the broker's own administrative one. A program documented as -// holding its own credential and given somebody else's is worse than one with no story at all. -func builderCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("builder issue", flag.ContinueOnError) - // Which machine will use it. Given, the credential is delivered by the mesh rather than - // printed for somebody to carry — which is the difference between the builder being a module - // and being a program somebody configures. - forNode := set.String("node", "", - "the machine that will run it, so the mesh delivers the credential instead of printing it") - module := set.String("module", "builder", "the module on that machine that will read it") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 2 || positionals[0] != "issue" { - return errors.New("builder issue [--node ]") - } - name := positionals[1] - - management, err := broker.ManagementFromEnvironment() - if err != nil { - return err - } - - // The same shape of secret a token carries: enough entropy that guessing is not a strategy, - // and safe to put in a URL because that is where it goes. - raw := make([]byte, 32) - if _, err := rand.Read(raw); err != nil { - return err - } - password := base64.RawURLEncoding.EncodeToString(raw) - if err := management.CreateBuilderAccount(ctx, name, password); err != nil { - return err - } - - fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n", - name, link.BuildQueue, link.Exchange) - - if *forNode != "" { - known, err := broker.FromEnvironment() - if err != nil { - return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err) - } - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - // The URL and what verifies the broker, together. A mesh's broker presents a certificate - // of the mesh's own, which is in no public trust store — so a URL on its own reaches only - // a broker somebody else vouches for, and the connection fails at TLS with an error about - // an unknown authority rather than about a missing pin. - // - // **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same - // way: out of band relative to the broker, so what is trusted does not come from the thing - // being trusted. - held, err := json.Marshal(struct { - URL string `json:"url"` - Fingerprint string `json:"fingerprint,omitempty"` - }{ - URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, known.Address), - Fingerprint: known.Fingerprint, - }) - if err != nil { - return err - } - if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil { - return err - } - // Not printed. It is sealed to that machine and the mesh cannot read it back, which is - // the whole point — printing it here would put the one copy that matters on a terminal. - fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n", - *forNode, *module) - fmt.Printf(" run `push %s` to send it\n", *forNode) - return nil - } - - // The whole line only when the address is known. A URL with a placeholder where the host - // should be is a URL somebody pastes and then debugs, and the placeholder is the last thing - // they look at. - if known, err := broker.FromEnvironment(); err == nil { - fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address) - } else { - fmt.Printf(" the password is %s\n\n", password) - fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+ - " Put the password in MESH_BROKER_AMQP on the build machine.\n\n", - broker.AddressVar) - } - // Shown once, like a token, and for the same reason: what is stored is the broker's own hash - // of it, and a control plane that could show it back would be a control plane that holds it. - fmt.Println("This is the only time it is shown.") - return nil -} - -// openLicences connects to the context that holds which model access exists and who may use it. -func openLicences(ctx context.Context) (*licences.Licences, error) { - held, err := licences.Open(ctx) - if err != nil { - return nil, err - } - if err := held.Ready(ctx, 30*time.Second); err != nil { - held.Close() - return nil, err - } - return held, nil -} - -// licencesFor is what this node can be answered with by record, and what it was put on. -// -// A mesh with no licences at all is the ordinary case and must not be an error: every existing -// mesh is one, and a control plane that refused to plan because nobody had bought an API key -// would be unusable for the thing it already does. -func licencesFor(ctx context.Context, node string) ( - map[string][]catalogue.Record, map[string]map[string]catalogue.Record, error) { - - held, err := openLicences(ctx) - if err != nil { - return nil, nil, err - } - defer held.Close() - - all, err := held.All(ctx) - if err != nil { - return nil, nil, err - } - if len(all) == 0 { - return nil, nil, nil - } - - offered := map[string][]catalogue.Record{} - byName := map[string]catalogue.Record{} - for _, one := range all { - record := catalogue.Record{Name: one.Name, Serves: one.Serves} - offered[licences.Provision] = append(offered[licences.Provision], record) - byName[one.Name] = record - } - - using := map[string]map[string]catalogue.Record{} - for _, one := range all { - holders, err := held.HoldersOf(ctx, one.Name) - if err != nil { - return nil, nil, err - } - for _, h := range holders { - if h.Node != node { - continue - } - if using[h.Module] == nil { - using[h.Module] = map[string]catalogue.Record{} - } - using[h.Module][licences.Provision] = byName[one.Name] - } - } - return offered, using, nil -} - -// keyFor is the licence key sealed to one machine, for one module. -// -// **Empty is not an error here.** The mesh discarded the plaintext when it was supplied, so a -// holder recorded afterwards genuinely has no key — and the declaration refuses that by name, -// where the module and the path are both in view, rather than here. -func keyFor(ctx context.Context, licence, node, module string) (string, error) { - held, err := openLicences(ctx) - if err != nil { - return "", err - } - defer held.Close() - return held.KeyFor(ctx, licence, node, module) -} - -// buildBehind builds every module the mesh holds older than its source has. -// -// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already -// records where each module came from and what its source last had; until this, a person read -// that list and retyped each repository — which is a person being the loop, and the thing a -// pipeline was doing before it was taken away. -// -// Each is built and recorded on its own. **One failing does not stop the others**, for the same -// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad -// repository holds back nine good ones is a mesh where nobody dares add the tenth. -func buildBehind(ctx context.Context, wait time.Duration) error { - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - - held, err := inv.Catalogued(ctx) - if err != nil { - return err - } - var stale []inventory.Entry - for _, e := range held { - if !e.Source.Current() { - stale = append(stale, e) - } - } - if len(stale) == 0 { - // Said rather than doing nothing quietly: "nothing needed building" and "this did not - // run" must never look the same. - fmt.Println("every module the mesh holds is what its source last had") - return nil - } - - fmt.Printf("%d module(s) behind their source:\n", len(stale)) - for _, e := range stale { - fmt.Printf(" %s %s < %s\n", - e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head)) - } - fmt.Println() - - var failed []string - for _, e := range stale { - fmt.Printf("--- %s\n", e.Manifest.Module) - // Its own recorded ref, not its head commit: a module tracking a branch should be built - // from that branch, and pinning to the commit the mesh happened to notice would quietly - // turn a tracked branch into a pin. - if err := buildOne(ctx, e.Source.Repository, e.Source.Ref, wait); err != nil { - fmt.Printf(" %v\n", err) - failed = append(failed, e.Manifest.Module) - } - } - - if len(failed) > 0 { - return fmt.Errorf("%d of %d could not be built: %s", - len(failed), len(stale), strings.Join(failed, ", ")) - } - fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n", - len(stale)) - return nil -} - -// buildOne asks a build machine for one repository and records everything that came back. -// -// Separated from the command so `--behind` can walk a list without a second path to the same act. -func buildOne(ctx context.Context, repository, ref string, wait time.Duration) error { - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - server, err := link.Connect(nil, nil) - if err != nil { - return err - } - defer server.Close() - - // Correlated by something the control plane makes, not by the module's name: two builds of one - // module can be in flight, and the second answer is not the first one's. - request := link.BuildRequest{ - ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), - Repository: repository, - Ref: ref, - } - fmt.Printf("asked for %s", request.Repository) - if ref != "" { - fmt.Printf(" at %s", ref) - } - fmt.Println() - - result, err := link.RequestBuild(ctx, server.Channel(), request, wait) - if err != nil { - return err - } - - // Kept before it is judged. A failed build that leaves no trace is indistinguishable from one - // nobody asked for, and the difference is the whole of whether somebody should be looking at - // something. - inv, err := openInventory(ctx) - if err != nil { - return err - } - defer inv.Close() - if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil { - return err - } - - if result.Failed != "" { - // The builder's own words. Wrapping them in something about the control plane would put - // two explanations between a person and a build log. - return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) - } - - for _, made := range result.Made { - fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference) - } - - // Parsed with the same parser a hand-written manifest goes through. A second path would be a - // second thing to disagree about what a manifest is. - manifest, err := catalogue.ParseManifest(result.Manifest) - if err != nil { - return fmt.Errorf("%s built %s and what came back is not a manifest: %w", - result.On, result.Repository, err) - } - - // Recorded with where it came from, so "is this current?" is answerable without building it - // again (novox/hq ADR 0009). - if err := inv.RegisterModule(ctx, manifest, inventory.Source{ - Repository: result.Repository, Ref: result.Ref, - BuiltFrom: result.Commit, Head: result.Commit, - }); err != nil { - return err - } - fmt.Printf("\n%s %s, built on %s from %s\n", - manifest.Module, manifest.Version, result.On, short(result.Commit)) - fmt.Printf(" run `assign %s` to put it somewhere\n", manifest.Module) - return nil -} - -// buildAndShow builds and prints the manifest without recording anything. -func buildAndShow(ctx context.Context, repository, ref string, wait time.Duration) error { - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - server, err := link.Connect(nil, nil) - if err != nil { - return err - } - defer server.Close() - - result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{ - ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Ref: ref, - }, wait) - if err != nil { - return err - } - if result.Failed != "" { - return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) - } - manifest, err := catalogue.ParseManifest(result.Manifest) - if err != nil { - return fmt.Errorf("%s built %s and what came back is not a manifest: %w", - result.On, result.Repository, err) - } - body, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil -} - -// answers is what the three questions came back with, read once. -type answers struct { - wrong []inventory.Doing - nodes []inventory.Node - quiet []inventory.Node - behind map[string][]string - sources map[string]inventory.Source - // waiting is every machine not running what the mesh would send it. - waiting []inventory.Machine -} - -// theThreeQuestions reads what anything answering "is the mesh alright" needs. -// -// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There -// are three now — a person's status, its JSON, and a page — and three implementations of "which -// machine is not doing what it was told" would be three chances to disagree about it. -// -// The order is the design and not a convenience: is anything broken, is anything not answering, is -// anything out of date. The first has consequences now, the second may, the third is a plan for -// later — and anything that led with the third would bury the first. -func theThreeQuestions(ctx context.Context, inv *inventory.Inventory) (answers, error) { - var out answers - var err error - - out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx) - if err != nil { - return answers{}, err - } - out.nodes, err = inv.Nodes(ctx) - if err != nil { - return answers{}, err - } - for _, n := range out.nodes { - // Never heard from, or not lately. Different from failing: a machine that says nothing - // may be new, switched off, or unreachable, and none of those is a machine that tried - // and could not. - if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour { - out.quiet = append(out.quiet, n) - } - } - out.behind, err = inv.Behind(ctx) - if err != nil { - return answers{}, err - } - // And which machines are not running what the mesh would send them. The same question as a - // module being behind its source, one level down: that one says the catalogue is out of date, - // this one says a machine is — and only the second has anybody's change waiting in it. - would, err := wouldSend(ctx, inv, out.nodes) - if err != nil { - return answers{}, err - } - out.waiting, err = inv.Waiting(ctx, would) - if err != nil { - return answers{}, err - } - out.sources = map[string]inventory.Source{} - for module := range out.behind { - from, err := inv.SourceOf(ctx, module) - if err != nil { - return answers{}, err - } - out.sources[module] = from - } - return out, nil -} - -// showNode says what one machine reported about itself, in its own words. -// -// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what -// a machine can do is the one it gave — and its detail is half of that account. The mesh was -// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with -// nowhere to go: a person told a machine lacks something wants to know what the detector saw. -// -// It is also where "what should it be configured as" is read. The same line that gates an -// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it. -func showNode(ctx context.Context, inv *inventory.Inventory, name string) error { - node, err := inv.NodeByName(ctx, name) - if err != nil { - return err - } - fmt.Printf("%s\n", node.Name) - fmt.Printf(" last heard from %s\n", heardFrom(node)) - - held, err := inv.Profile(ctx, name) - if err != nil { - return err - } - if held == nil { - // Never reported is not the same as reported nothing, and the remedy differs: one is a - // machine that has not run the host yet, the other is a machine that ran it and can do - // nothing. - fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" + - " capability is refused here — run the host on it\n") - return nil - } - if len(held) == 0 { - fmt.Printf("\n it reported no capabilities at all\n") - return nil - } - - fmt.Printf("\n what it can do, as it reported:\n") - for _, c := range held { - mark := "no " - if c.Present { - mark = "yes" - } - fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail) - } - - assigned, err := inv.Assigned(ctx, name) - if err != nil { - return err - } - if len(assigned) > 0 { - fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) - } - return nil -} - -// digestOf is what the mesh compares to answer "has this machine been sent what it should be". -// -// Over the same bytes that are sent, so the comparison is of the thing itself rather than of -// something derived beside it that could drift from it. -func digestOf(body []byte) string { - sum := sha256.Sum256(body) - return hex.EncodeToString(sum[:]) -} - -// wouldSend is the digest of what each machine should be right now. -// -// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot -// be worked out" is a different problem with a different remedy, and `plan` is where it is said. -func wouldSend(ctx context.Context, inv *inventory.Inventory, - nodes []inventory.Node) (map[string]string, error) { - - gens, err := generators(ctx, inv) - if err != nil { - return nil, err - } - out := map[string]string{} - for _, n := range nodes { - plan, settings, err := planFor(ctx, inv, n.Name) - if err != nil { - continue - } - resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) - if err != nil { - continue - } - body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources}) - if err != nil { - return nil, err - } - out[n.Name] = digestOf(body) - } - return out, nil -} - -// namesInTheMesh is every machine's internal name and the address behind it. -// -// A machine with no address has no name: writing one that resolves to nothing is worse than not -// writing it, because a connection to an address that does not answer hangs where a name that -// does not resolve fails at once and says so. That is the rule the hosts file already follows, -// and this is the same set read the same way. -func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) { - places, err := inv.Overlays(ctx) - if err != nil { - return nil, err - } - out := map[string]string{} - for _, p := range places { - if strings.TrimSpace(p.Address) == "" { - continue - } - out[overlay.InternalName(p.Name)] = p.Address - } - return out, nil -} diff --git a/cmd/mesh-control/modules.go b/cmd/mesh-control/modules.go new file mode 100644 index 0000000..54da37d --- /dev/null +++ b/cmd/mesh-control/modules.go @@ -0,0 +1,350 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "sort" + "strings" + + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/overlay" +) + +// the catalogue: what exists, what is assigned, and how it is configured. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// provided is what comes with the control plane rather than from a repository. +// +// WireGuard, the names, and the domain module over both. The first two are here because the code +// that works out their files is here: +// a peer list is derived from every machine at once, so it cannot be written in a manifest, and +// whatever computes it has to live wherever the whole picture is. +// +// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent +// from a machine nobody gave it to. +func providedModules() []catalogue.Manifest { + var out []catalogue.Manifest + for _, raw := range []map[string]any{ + overlay.Manifest(), overlay.NamesManifest(), overlay.ResolverManifest(), + overlay.DomainManifest(), + } { + var m catalogue.Manifest + b, _ := json.Marshal(raw) + _ = json.Unmarshal(b, &m) + out = append(out, m) + } + return out +} + +var provided = providedModules() + +func moduleCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("module add , module list, or module forget ") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + switch args[0] { + case "add": + set := flag.NewFlagSet("module add", flag.ContinueOnError) + repo := set.String("source", "", "where this module comes from") + ref := set.String("ref", "", "the branch followed there") + commit := set.String("commit", "", "the commit this manifest was read at") + positionals, err := parseAround(set, args[1:]) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("module add [--source --ref --commit ]") + } + raw, err := os.ReadFile(positionals[0]) + if err != nil { + return err + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + return err + } + // Provenance together or not at all. A source with no commit cannot be compared against + // anything, so it would record where the module came from and still never be able to say + // the mesh is behind it — which is the one thing recording it is for. + if (*repo == "") != (*commit == "") { + return errors.New("--source and --commit go together: a source with no commit " + + "cannot be compared against anything, and a commit with no source has nothing " + + "to be compared with") + } + if err := inv.RegisterModule(ctx, m, inventory.Source{ + Repository: *repo, Ref: *ref, BuiltFrom: *commit, + }); err != nil { + return err + } + fmt.Printf("%s registered", m.Module) + if *commit != "" { + fmt.Printf(" from %s", short(*commit)) + } + if len(m.Provides) > 0 { + fmt.Printf(", providing %s", describeOffers(m.Provides)) + } + fmt.Println() + for _, c := range m.Claims { + fmt.Printf(" claims %s, one per %s\n", c.Name, c.At()) + } + return nil + + case "list": + // The catalogue: what exists, where it came from, whether it is current, and who runs it. + // The provenance was recorded from the first build and nothing showed it, which made + // "is this current?" a question you could only answer by reading the database. + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + if len(entries) == 0 { + fmt.Println("this mesh knows about no modules yet") + return nil + } + var stale int + for _, e := range entries { + m := e.Manifest + fmt.Printf("%-18s %-8s", m.Module, m.Version) + + switch { + case e.Provided: + fmt.Printf(" %-22s", "with the control plane") + case e.Source.Repository == "": + // Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and + // worth saying, because nothing can rebuild it. + fmt.Printf(" %-22s", "handed over") + case !e.Source.Current(): + stale++ + fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head)) + default: + fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom)) + } + + if len(e.On) > 0 { + fmt.Printf(" on %s", strings.Join(e.On, ", ")) + } else { + fmt.Printf(" on nothing") + } + fmt.Println() + + var says []string + if len(m.Provides) > 0 { + says = append(says, "provides "+describeOffers(m.Provides)) + } + if len(m.Requires) > 0 { + says = append(says, "requires "+strings.Join(m.Requires, ", ")) + } + for _, c := range m.Claims { + says = append(says, "claims "+c.At()+"/"+c.Name) + } + if len(m.Capabilities) > 0 { + says = append(says, "needs "+strings.Join(m.Capabilities, ", ")) + } + if len(says) > 0 { + fmt.Printf(" %s\n", strings.Join(says, " · ")) + } + } + if stale > 0 { + fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale) + } + return nil + + case "moved": + if len(args) != 3 { + return errors.New("module moved — the source has a newer commit") + } + if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil { + return err + } + from, err := inv.SourceOf(ctx, args[1]) + if err != nil { + return err + } + if from.Current() { + fmt.Printf("%s is current at %s\n", args[1], short(from.Head)) + return nil + } + fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n", + args[1], short(from.BuiltFrom), short(from.Head)) + fmt.Printf(" run `build %s` to catch up\n", from.Repository) + return nil + + case "forget": + if len(args) != 2 { + return errors.New("module forget ") + } + if err := inv.ForgetModule(ctx, args[1]); err != nil { + return err + } + fmt.Printf("%s forgotten\n", args[1]) + return nil + + default: + return fmt.Errorf("module has no %q; it has add, list, moved and forget", args[0]) + } +} + +func assignCommand(ctx context.Context, verb string, args []string) error { + if len(args) != 2 { + return fmt.Errorf("%s ", verb) + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + if verb == "unassign" { + if err := inv.Unassign(ctx, args[0], args[1]); err != nil { + return err + } + fmt.Printf("%s no longer runs %s — run `push %s` to make it so\n", args[0], args[1], args[0]) + return nil + } + if err := inv.Assign(ctx, args[0], args[1]); err != nil { + return err + } + fmt.Printf("%s is assigned %s\n", args[0], args[1]) + + // Resolved immediately, because an assignment that cannot be applied should be said now + // rather than at the next push. The assignment is kept either way: it is what a person meant, + // and the refusal is about the set rather than about this one. + if _, _, err := planFor(ctx, inv, args[0]); err != nil { + fmt.Println() + return err + } + fmt.Printf(" run `push %s` to send it\n", args[0]) + return nil +} + +func settingsCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("settings set [--node ], or settings clear [--node ]") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + set := flag.NewFlagSet("settings", flag.ContinueOnError) + node := set.String("node", "", "one machine, rather than the whole mesh") + positionals, err := parseAround(set, args[1:]) + if err != nil { + return err + } + + where := "the whole mesh" + if *node != "" { + where = *node + } + + switch args[0] { + case "set": + if len(positionals) != 2 { + return errors.New("settings set [--node ]") + } + raw, err := os.ReadFile(positionals[1]) + if err != nil { + return err + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + return fmt.Errorf("%s is not a settings file: %w", positionals[1], err) + } + if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil { + return err + } + + var keys []string + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", ")) + fmt.Println(" run `push` to send it") + return nil + + case "clear": + if len(positionals) != 1 { + return errors.New("settings clear [--node ]") + } + if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil { + return err + } + fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where) + return nil + + default: + return fmt.Errorf("settings has no %q; it has set and clear", args[0]) + } +} + +// describeOffers says what a module provides, and marks the ones answered from anywhere in the +// mesh — because "provides a database" and "provides a shell" are read the same way and mean +// entirely different things about where the answer has to be. +func describeOffers(offers []catalogue.Offer) string { + var out []string + for _, o := range offers { + if o.At() == catalogue.ScopeMesh { + out = append(out, o.Name+" (from anywhere in the mesh)") + continue + } + out = append(out, o.Name) + } + return strings.Join(out, ", ") +} + +// pinCommand says which node a machine gets a provision from. +// +// Needed only when more than one could answer, and recordable before that -- a mesh with one +// database should not change where an existing machine gets its data the day a second one +// arrives. +func pinCommand(ctx context.Context, args []string, setting bool) error { + if setting && len(args) != 3 { + return errors.New("pin ") + } + if !setting && len(args) != 2 { + return errors.New("unpin ") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + if !setting { + if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil { + return err + } + fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1]) + return nil + } + if args[0] == args[2] { + // Allowed by nothing here, and worth saying rather than resolving into a confusing + // refusal later: a node providing something to itself is a node-scoped provision, and + // this field is for the other kind. + return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+ + "provides, which does not need saying", args[0], args[1]) + } + if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil { + return err + } + fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2]) + fmt.Printf(" run `push %s` to send it\n", args[0]) + return nil +} diff --git a/cmd/mesh-control/network.go b/cmd/mesh-control/network.go new file mode 100644 index 0000000..0c59deb --- /dev/null +++ b/cmd/mesh-control/network.go @@ -0,0 +1,358 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/overlay" +) + +// the private network: who is on it, where, and what they are called. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +func overlayCIDR() string { + if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" { + return v + } + return "10.42.0.0/16" +} + +func overlayCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("overlay place [flags], or overlay show") + } + // Answered before anything is opened. A message about which command to use should not need a + // database to say so, and needing one turns a redirect into a connection error. + if args[0] == "push" { + return errors.New("`overlay push` is now `push`, which sends a node its network AND " + + "what its assignments resolve to — the two are computed from one picture of the " + + "mesh, and sending them separately would let them disagree") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + switch args[0] { + case "place": + return overlayPlace(ctx, inv, args[1:]) + case "show": + return overlayShow(ctx, inv) + + default: + return fmt.Errorf("overlay has no %q; it has place and show", args[0]) + } +} + +func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error { + if len(args) == 0 { + return errors.New("overlay place [--endpoint host:port] [--site name] [--hub]") + } + node := args[0] + + set := flag.NewFlagSet("overlay place", flag.ContinueOnError) + endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere") + site := set.String("site", "", "where this machine physically is, or empty if it roams") + hub := set.Bool("hub", false, "this node is the hub every other routes through") + if err := set.Parse(args[1:]); err != nil { + return err + } + + // Declared, all three. The address is evidence of reachability and is not the fact, and hub + // election by address prefix fails silently (novox/hq ADR 0007). + if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil { + return err + } + found, err := inv.NodeByName(ctx, node) + if err != nil { + return err + } + address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR()) + if err != nil { + return err + } + + fmt.Printf("%s is at %s on the overlay\n", node, address) + switch { + case *hub: + fmt.Println(" the hub — every node not sharing a site routes through it") + case *endpoint == "": + fmt.Println(" not dialable — it opens every path itself") + } + if *site != "" { + fmt.Printf(" at %s, so it peers directly with anything else there\n", *site) + } + return nil +} + +// network builds the private network over the machines that resolved the module for it. +// +// Not over every node the mesh knows. **A machine is on the private network because it was given +// the module**, and one that was not is absent from every peer list and from the names — which is +// the only thing "not on the network" can mean. Until this, having an address was enough, and +// there was no way to keep a machine off. +// +// Every node at once, which is the whole reason this is the control plane's work: a peer list is +// derived from all the others, so no node could compute its own. +func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, + refused map[string]string) (*overlay.Generator, error) { + places, err := inv.Overlays(ctx) + if err != nil { + return nil, err + } + nodes := make([]overlay.Node, 0, len(places)) + for _, p := range places { + if !on[p.Name] { + continue + } + nodes = append(nodes, overlay.Node{ + Name: p.Name, Key: p.Key, Endpoint: p.Endpoint, + Site: p.Site, Hub: p.Hub, Address: p.Address, + }) + } + if len(nodes) == 0 { + // Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this + // answers rather than refusing -- Compute would refuse for want of a hub, and reporting + // "no hub" to somebody who never asked for a network would be a lie about the cause. + return overlay.Empty(), nil + } + g, err := overlay.From(nodes, overlayCIDR(), "") + if err != nil && len(refused) > 0 { + // The network is missing something, and some machines could not be resolved at all. Those + // are almost always the same fact: a node that does not resolve contributes nothing, so + // reporting "no hub" would name a consequence and hide the cause. + var who []string + for name, why := range refused { + who = append(who, fmt.Sprintf(" %s: %s", name, why)) + } + sort.Strings(who) + return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+ + "probably why:\n%s", err, len(refused), strings.Join(who, "\n")) + } + return g, err +} + +// graph is the whole mesh's network, for showing it. +func graph(ctx context.Context, inv *inventory.Inventory) ([]overlay.Node, overlay.Graph, error) { + on, refused, err := whoResolves(ctx, inv, overlay.Requirement) + if err != nil { + return nil, nil, err + } + g, err := network(ctx, inv, on, refused) + if err != nil { + return nil, nil, err + } + return g.Nodes(), g.Graph(), nil +} + +// whoResolves is the machines whose resolution answers a requirement, and why the others did not. +// +// By what a module **provides**, not by its name. WireGuard is one way to have a private network +// and there could be others, so a machine is on the network because something it runs provides +// one — asking for a particular module by name would be the mistake this whole mechanism exists +// to avoid. +// +// Resolved rather than read from the assignment table, because a module can arrive by being +// required by something else, and a machine that needs the private network to do its job is on it +// for the same reason as one that was handed it directly. +func whoResolves(ctx context.Context, inv *inventory.Inventory, requirement string) ( + map[string]bool, map[string]string, error) { + nodes, err := inv.Nodes(ctx) + if err != nil { + return nil, nil, err + } + on := map[string]bool{} + // Why a node could not be resolved, kept rather than raised: one broken node must not stop + // the rest being described, and whoever is rendering that node will raise it themselves. + refused := map[string]string{} + for _, n := range nodes { + plan, _, err := planFor(ctx, inv, n.Name) + if err != nil { + refused[n.Name] = err.Error() + continue + } + for _, m := range plan.Modules { + for _, offered := range m.Offers() { + if offered == requirement { + on[n.Name] = true + } + } + } + } + return on, refused, nil +} + +// rendering is everything a declaration needs, computed over the whole mesh. +func generators(ctx context.Context, inv *inventory.Inventory) ( + map[string]catalogue.Generator, error) { + on, refused, err := whoResolves(ctx, inv, overlay.Addressing) + if err != nil { + return nil, err + } + net, err := network(ctx, inv, on, refused) + if err != nil { + return nil, err + } + // Both generators see the same machines: the ones on the private network. Names for a machine + // that is not on it would resolve to addresses it cannot reach, which is worse than no names. + return map[string]catalogue.Generator{ + overlay.Name: net, + overlay.Names: overlay.NamesFor(net.Nodes()), + overlay.Resolver: overlay.ResolverFor(net.Nodes()), + }, nil +} + +func overlayShow(ctx context.Context, inv *inventory.Inventory) error { + nodes, computed, err := graph(ctx, inv) + if err != nil { + return err + } + if len(nodes) == 0 { + // Not "this mesh has no nodes", which it said until the network became a module and was + // then a lie about the cause: a mesh can have every node it will ever have and nobody on + // the private network, because nobody asked for one. + fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n", + overlay.Name) + return nil + } + + for _, n := range nodes { + place := n.Address + if place == "" { + // Said, not skipped. A node with no place is a node with no network, and it should + // be visible here rather than quietly absent from a list of who is on it. + place = "no address — run `overlay place`" + } + fmt.Printf("%-16s %-14s", n.Name, place) + switch { + case n.Hub: + fmt.Print(" hub") + case !n.Reachable(): + fmt.Print(" not dialable") + } + if n.Site != "" { + fmt.Printf(" at %s", n.Site) + } + fmt.Println() + for _, p := range computed[n.Name] { + fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why) + } + } + return nil +} + +// SilentFor is how long a node may be quiet before the mesh says so. +// +// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number +// is not the point — being able to say "out of touch" at all is, and nothing could before. +const SilentFor = 3 * time.Minute + +// whereEveryoneIs is each machine's name on the private network, for the ones on it. +// +// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every +// other path here answers a question about one node by resolving the others; this one is called +// *from* that path, so doing the same would not terminate — which it did not, for two minutes, +// until it was run. +// +// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the +// private network depends on what it was assigned and what that requires, both of which are local +// facts. What it takes *from* other machines does not change the answer. +// +// The distinction that matters is kept: a machine absent from the network module's own view is +// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the +// network became something a machine is given. +func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory, + shelf map[string]catalogue.Manifest) (map[string]string, error) { + + if shelf == nil { + // Refused rather than answered. Being on the private network is a conclusion about what a + // node resolves to, so with no catalogue nothing resolves and the honest answer is + // "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused + // every certificate the mesh was asked for while saying the machine was on no network. + return nil, errors.New( + "asked where everyone is without the catalogue, which cannot be answered") + } + places, err := inv.Overlays(ctx) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, p := range places { + if p.Address == "" { + continue + } + assigned, err := inv.Assigned(ctx, p.Name) + if err != nil || len(assigned) == 0 { + continue + } + caps, _ := inv.ProfileOf(ctx, p.Name) + got, err := catalogue.Resolve(shelf, assigned, + catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps}, + catalogue.World{Unchecked: true}) + if err != nil { + continue + } + for _, m := range got.Modules { + for _, offered := range m.Offers() { + if offered == overlay.Requirement { + out[p.Name] = overlay.InternalName(p.Name) + } + } + } + } + return out, nil +} + +// onThePrivateNetwork is every node's address on the overlay, sorted. +// +// A node with no address is left out rather than rendered as an empty source: an empty entry in a +// source set is a syntax error in the rule file, and a rule file that does not load leaves the +// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule, +// because nothing reports it. +func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory) ([]string, error) { + places, err := inv.Overlays(ctx) + if err != nil { + return nil, err + } + var out []string + for _, p := range places { + if strings.TrimSpace(p.Address) != "" { + out = append(out, p.Address) + } + } + sort.Strings(out) + return out, nil +} + +// namesInTheMesh is every machine's internal name and the address behind it. +// +// A machine with no address has no name: writing one that resolves to nothing is worse than not +// writing it, because a connection to an address that does not answer hangs where a name that +// does not resolve fails at once and says so. That is the rule the hosts file already follows, +// and this is the same set read the same way. +func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) { + places, err := inv.Overlays(ctx) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, p := range places { + if strings.TrimSpace(p.Address) == "" { + continue + } + out[overlay.InternalName(p.Name)] = p.Address + } + return out, nil +} diff --git a/cmd/mesh-control/nodes.go b/cmd/mesh-control/nodes.go new file mode 100644 index 0000000..bbfc053 --- /dev/null +++ b/cmd/mesh-control/nodes.go @@ -0,0 +1,296 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-control/internal/broker" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-control/internal/token" +) + +// what a machine is, and what it is allowed to be told. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +func nodeCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("node add , node list, or node show ") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + switch args[0] { + case "show": + if len(args) != 2 { + return errors.New("node show ") + } + return showNode(ctx, inv, args[1]) + case "add": + if len(args) != 2 { + return errors.New("node add ") + } + node, err := inv.AddNode(ctx, args[1]) + if err != nil { + return err + } + fmt.Printf("added %s (%s)\n", node.Name, node.ID) + return nil + + case "list": + nodes, err := inv.Nodes(ctx) + if err != nil { + return err + } + if len(nodes) == 0 { + // Said rather than printed as nothing: an empty list and a failed read must never + // look the same, and this command answering "none" is only honest because getting + // here means the store answered. + fmt.Println("this mesh has no node records yet") + return nil + } + for _, n := range nodes { + fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID) + } + return nil + + default: + return fmt.Errorf("node has no %q; it has add and list", args[0]) + } +} + +func tokenCommand(ctx context.Context, args []string) error { + if len(args) == 0 || args[0] != "issue" { + return errors.New("token issue --node , or token issue --new ") + } + + set := flag.NewFlagSet("token issue", flag.ContinueOnError) + existing := set.String("node", "", "issue for a node record that already exists") + fresh := set.String("new", "", "create the node record, then issue for it") + validFor := set.Duration("for", time.Hour, "how long the token may be used") + if err := set.Parse(args[1:]); err != nil { + return err + } + + // Exactly one, because the difference is what the token binds to. A command that guessed + // would sometimes create a second record for a machine that already has one. + if (*existing == "") == (*fresh == "") { + return errors.New("give exactly one of --node or --new : the first is a " + + "machine the mesh already has a record for, the second is one it has never seen") + } + + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + name := *existing + if *fresh != "" { + node, err := inv.AddNode(ctx, *fresh) + if err != nil { + return err + } + name = node.Name + } + + issued, err := inv.IssueToken(ctx, name, *validFor) + if err != nil { + return err + } + + // Assembled from two contexts by the process that holds both grants. Neither reads the + // other's store (novox/hq ADR 0008) — each is asked for its own part. + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + key, err := ident.Establish(ctx) + if err != nil { + return err + } + + // The account is created before the token is handed over, which is what removes the + // chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can + // exist before it does. The one-time secret IS the password, so a node's first connection is + // already authenticated and enrolment is what happens over it. + if management, err := broker.ManagementFromEnvironment(); err == nil { + if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil { + return err + } + fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n", + issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange) + } else if !errors.Is(err, broker.ErrNotConfigured) { + return err + } + + made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret} + + // Absent is a state, not a failure: a control plane can hold records and a key before it has + // a broker. What it cannot do is issue a token anybody could use, and Missing() says so. + known, err := broker.FromEnvironment() + switch { + case err == nil: + made.Broker, made.Fingerprint = known.Address, known.Fingerprint + case errors.Is(err, broker.ErrNotConfigured): + default: + return err + } + encoded, err := made.Encode() + if err != nil { + return err + } + + fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n", + issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded) + fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.") + + if missing := made.Missing(); len(missing) > 0 { + fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n") + for _, m := range missing { + fmt.Printf(" - %s\n", m) + } + fmt.Printf("\nSet %s and %s once the broker is raised.\n", + broker.AddressVar, broker.CertificateVar) + } + return nil +} + +func identityCommand(ctx context.Context, args []string) error { + if len(args) == 0 || args[0] != "show" { + return errors.New("identity show") + } + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + // Establish rather than read: a control plane asked for its identity before it has one should + // get one, not an error. Generating it is idempotent, so this is safe to run at any time. + key, err := ident.Establish(ctx) + if err != nil { + return err + } + fmt.Printf("signing key %s\n", key.ID) + fmt.Printf("fingerprint %s\n", key.Fingerprint()) + fmt.Printf("created %s\n", key.Created.Format(time.RFC3339)) + fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" + + "declaration because it carries a signature this key made (novox/hq ADR 0004).\n") + return nil +} + +func brokerCommand(args []string) error { + if len(args) == 0 || args[0] != "show" { + return errors.New("broker show") + } + known, err := broker.FromEnvironment() + if errors.Is(err, broker.ErrNotConfigured) { + fmt.Printf("no broker configured. Set %s and %s.\n\n"+ + "Until then tokens carry the signing key and the one-time secret, and say what they\n"+ + "are missing. They cannot be used to join.\n", + broker.AddressVar, broker.CertificateVar) + return nil + } + if err != nil { + return err + } + fmt.Printf("address %s\n", known.Address) + fmt.Printf("fingerprint %s\n", known.Fingerprint) + fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" + + "node checks it before sending anything (novox/hq ADR 0004).\n") + return nil +} + +// heardFrom says when a node was last heard from, in a form somebody can act on. +// +// "never" and "an hour ago" are different answers and are kept different. A node that has never +// spoken did not finish joining; a node last heard from an hour ago is running an hour-old +// picture of the mesh. +func heardFrom(n inventory.Node) string { + silent, ever := n.Silent() + switch { + case !ever: + return "never spoken" + case silent > SilentFor: + return "out of touch " + roughly(silent) + default: + return "here" + } +} + +// roughly is a duration a person reads rather than parses. +func roughly(d time.Duration) string { + switch { + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} + +// showNode says what one machine reported about itself, in its own words. +// +// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what +// a machine can do is the one it gave — and its detail is half of that account. The mesh was +// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with +// nowhere to go: a person told a machine lacks something wants to know what the detector saw. +// +// It is also where "what should it be configured as" is read. The same line that gates an +// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it. +func showNode(ctx context.Context, inv *inventory.Inventory, name string) error { + node, err := inv.NodeByName(ctx, name) + if err != nil { + return err + } + fmt.Printf("%s\n", node.Name) + fmt.Printf(" last heard from %s\n", heardFrom(node)) + + held, err := inv.Profile(ctx, name) + if err != nil { + return err + } + if held == nil { + // Never reported is not the same as reported nothing, and the remedy differs: one is a + // machine that has not run the host yet, the other is a machine that ran it and can do + // nothing. + fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" + + " capability is refused here — run the host on it\n") + return nil + } + if len(held) == 0 { + fmt.Printf("\n it reported no capabilities at all\n") + return nil + } + + fmt.Printf("\n what it can do, as it reported:\n") + for _, c := range held { + mark := "no " + if c.Present { + mark = "yes" + } + fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail) + } + + assigned, err := inv.Assigned(ctx, name) + if err != nil { + return err + } + if len(assigned) > 0 { + fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) + } + return nil +} diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go new file mode 100644 index 0000000..a206690 --- /dev/null +++ b/cmd/mesh-control/plan.go @@ -0,0 +1,556 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-control/internal/catalogue" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/licences" +) + +// working out what one machine should be. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// planFor works out everything a node should run, from what was assigned to it. +func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return catalogue.Resolution{}, nil, err + } + assigned, err := inv.Assigned(ctx, nodeName) + if err != nil { + return catalogue.Resolution{}, nil, err + } + capabilities, err := inv.ProfileOf(ctx, nodeName) + if err != nil { + return catalogue.Resolution{}, nil, err + } + + places, err := inv.Overlays(ctx) + if err != nil { + return catalogue.Resolution{}, nil, err + } + var site string + for _, p := range places { + if p.Name == nodeName { + site = p.Site + } + } + + world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName) + if err != nil { + return catalogue.Resolution{}, nil, err + } + world.Pinned, err = inv.PinsFor(ctx, nodeName) + if err != nil { + return catalogue.Resolution{}, nil, err + } + + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return catalogue.Resolution{}, nil, err + } + + // What this mesh can answer with a record rather than a machine, and which record each of + // this node's modules was put on. Read across a context boundary by name, which is what + // crossing one is allowed to carry (novox/hq ADR 0008). + world.Licences, world.Using, err = licencesFor(ctx, nodeName) + if err != nil { + return catalogue.Resolution{}, nil, err + } + + resolved, err := catalogue.Resolve(shelf, assigned, + catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities, + At: onNetwork[nodeName]}, world) + if err != nil { + return catalogue.Resolution{}, nil, err + } + + // The credential for each thing this node takes from elsewhere. Made once and kept, so the + // password a provider is told to create is the one its consumer was given — and sealed to + // this node before it was ever written down, so nothing between here and there can read it. + for i, n := range resolved.Needs { + if n.ByRecord { + // Answered by something the mesh holds, so there is no pair-wise secret between two + // machines. Its key was supplied by a person and sealed to this node then; the mesh + // discarded the plaintext and cannot make another. + sealed, err := keyFor(ctx, n.From, nodeName, n.For) + if err != nil { + return catalogue.Resolution{}, nil, err + } + resolved.Needs[i].Sealed = sealed + continue + } + secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.From) + if err != nil { + // Said rather than skipped. A machine that resolves cleanly and receives no + // credential is one that will fail to authenticate at some later, less obvious + // moment. + return catalogue.Resolution{}, nil, fmt.Errorf( + "%s needs %s from %s and no credential could be made for it: %w", + nodeName, n.Name, n.From, err) + } + resolved.Needs[i].Sealed = secret.ForConsumer + } + + // Settings for everything that resolved, including modules nobody assigned directly: a + // requirement pulled in by something else is still configurable, and finding out that it is + // not only when you try would be an arbitrary line nobody could predict. + settings := catalogue.SettingsBy{} + var stray []string + for _, m := range resolved.Modules { + layers, err := inv.SettingsFor(ctx, nodeName, m.Module) + if err != nil { + return catalogue.Resolution{}, nil, err + } + if len(layers) == 0 { + continue + } + settings[m.Module] = layers + stray = append(stray, catalogue.UnusedSettings(m, layers)...) + } + if len(stray) > 0 { + // Somebody set something that reaches no file. Said here rather than discovered by the + // machine not behaving differently, which is the slowest way there is. + return catalogue.Resolution{}, nil, fmt.Errorf( + "these settings reach nothing:\n - %s", strings.Join(stray, "\n - ")) + } + return resolved, settings, nil +} + +// theRestOfTheMesh is what every other node holds and offers. +// +// Two things at once because they come from the same place — resolving the other nodes — and +// because both are facts about what is actually running rather than records that could disagree +// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node +// runs; neither is a table somebody keeps up to date. +// +// **Two passes over the others.** What a node offers the mesh needs that node resolved, and +// resolving it may need what the mesh offers. So the first pass takes brokered requirements on +// trust and answers only *what does each node offer*; the second answers everything with that in +// hand. Nothing is ever declared from the first. +func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, + shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) { + + // Every node, not only the placed ones. A machine that was never put on the private network + // still runs modules, still holds claims, and still offers whatever it offers. + nodes, err := inv.Nodes(ctx) + if err != nil { + return catalogue.World{}, err + } + places, err := inv.Overlays(ctx) + if err != nil { + return catalogue.World{}, err + } + siteOf := map[string]string{} + for _, p := range places { + siteOf[p.Name] = p.Site + } + // Which machines are actually on the private network, and what they are called there. Not + // "has an address" — that was true of every placed machine and told you nothing about whether + // anything could reach it. It is what resolved the module. + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return catalogue.World{}, err + } + + type candidate struct { + node catalogue.Node + assigned []string + } + var others []candidate + for _, n := range nodes { + if n.Name == exclude { + continue + } + theirs, err := inv.Assigned(ctx, n.Name) + if err != nil || len(theirs) == 0 { + continue + } + caps, _ := inv.ProfileOf(ctx, n.Name) + others = append(others, candidate{ + catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps, + At: onNetwork[n.Name]}, theirs}) + } + + offered := map[string][]catalogue.Provider{} + for _, o := range others { + got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true}) + if err != nil { + // Their set does not resolve for some other reason. Not this node's problem to + // report, and nothing of theirs is running, so it offers nothing. + continue + } + for _, m := range got.Modules { + for _, name := range m.OffersAt(catalogue.ScopeMesh) { + // What that module says a consumer needs to know, with that node's settings on + // it: a port somebody moved on the provider is a port its consumers must be told + // about, and the two coming from different places is how they come to disagree. + serves := m.Serves[name] + if len(serves) > 0 { + layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module) + if err != nil { + return catalogue.World{}, err + } + serves, err = catalogue.Settle(serves, layers) + if err != nil { + return catalogue.World{}, err + } + } + offered[name] = append(offered[name], catalogue.Provider{ + Node: o.node.Name, At: o.node.At, Serves: serves}) + } + } + } + for k := range offered { + sort.Slice(offered[k], func(i, j int) bool { + return offered[k][i].Node < offered[k][j].Node + }) + } + + world := catalogue.World{Offered: offered} + for _, o := range others { + got, err := catalogue.Resolve(shelf, o.assigned, o.node, world) + if err != nil { + continue + } + world.Held = append(world.Held, got.Claims...) + } + return world, nil +} + +// declarationFor is everything a node would be sent. +// +// One place, because there were three and one of them was written before credentials existed and +// silently produced a declaration missing them — a difference between what `plan` showed and what +// `plan --json` handed to anything reading it. +func declarationFor(ctx context.Context, inv *inventory.Inventory, node string, + plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) { + gens, err := generators(ctx, inv) + if err != nil { + return nil, err + } + return declarationWith(ctx, inv, node, plan, settings, gens) +} + +// declarationWith is the same, for a caller that has already worked out the generators once and +// is about to use them for every node. +func declarationWith(ctx context.Context, inv *inventory.Inventory, node string, + plan catalogue.Resolution, settings catalogue.SettingsBy, + gens map[string]catalogue.Generator) ([]map[string]any, error) { + grants, err := grantsFor(ctx, inv, node) + if err != nil { + return nil, err + } + // And each module's own secrets — a superuser password, an administrator, an account. Made + // per node, so a module running on three machines has three. + needed := map[string]map[string]string{} + for _, m := range plan.Modules { + for name := range m.Needs { + sealed, err := inv.SecretForModule(ctx, node, m.Module, name) + if err != nil { + return nil, err + } + if needed[m.Module] == nil { + needed[m.Module] = map[string]string{} + } + needed[m.Module][name] = sealed + } + } + // And a certificate for this machine's name inside the mesh, when anything on it asks. Issued + // rather than stored: the node's key does not change, so signing again produces an equally + // valid certificate and there is nothing to keep in step. + var certificate, authority string + for _, m := range plan.Modules { + if m.Certificate == nil { + continue + } + issued, meshCA, err := certificateFor(ctx, inv, node) + if err != nil { + return nil, err + } + certificate, authority = issued, meshCA + break + } + + // And who else is on the private network, which is what a rule saying "from the mesh" + // resolves to. Every node's address, including this one's: a machine reaching itself by its + // own overlay address rather than by loopback is ordinary, and leaving it out would filter + // the node's own traffic to itself with no rule naming why. + private, err := onThePrivateNetwork(ctx, inv) + if err != nil { + return nil, err + } + + // And every machine's name, so a container can reach one. The same set that writes the + // machine's own hosts file — one reading, so a container and its machine cannot disagree + // about where another machine is. + names, err := namesInTheMesh(ctx, inv) + if err != nil { + return nil, err + } + + return plan.Declaration(catalogue.Rendering{ + Settings: settings, Generators: gens, Grants: grants, Needed: needed, + Certificate: certificate, Authority: authority, Mesh: private, Names: names}) +} + +// certificateFor is what the mesh certifies about one machine's internal name. +// +// It reaches across two contexts and reads neither one's store from the other: `inventory` knows +// the machine and whether it is on the private network, `identity` holds the authority and the +// key that machine reported. The process holding both grants asks each for its part +// (novox/hq ADR 0008). +func certificateFor(ctx context.Context, inv *inventory.Inventory, node string) (string, string, error) { + ident, err := openIdentity(ctx) + if err != nil { + return "", "", err + } + defer ident.Close() + + record, err := inv.NodeByName(ctx, node) + if err != nil { + return "", "", err + } + serving, err := ident.ServingKeyOf(ctx, record.ID) + if err != nil { + return "", "", err + } + if serving == "" { + // The machine joined before it had one, or never reported it. Said plainly, because the + // remedy is on the machine and no amount of pushing from here will produce one. + return "", "", fmt.Errorf( + "%s wants a certificate and has never told the mesh what key it serves with; it "+ + "joins again to report one", node) + } + + // The name it is certified for. Only a machine on the private network has one — a certificate + // for a name nothing resolves is a certificate nothing can check. + // + // With the catalogue, not without it. Being on the private network is a conclusion about what + // a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no + // network — which refused every certificate the mesh was asked for, and said the machine was + // not on a network it plainly was. + shelf, err := inv.Catalogue(ctx) + if err != nil { + return "", "", err + } + where, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return "", "", err + } + name := where[node] + if name == "" { + return "", "", fmt.Errorf( + "%s wants a certificate and is not on the private network, so it has no name inside "+ + "the mesh to be certified for", node) + } + + issued, err := ident.Certify(ctx, node, name, serving) + if err != nil { + return "", "", err + } + authority, err := ident.EstablishAuthority(ctx) + if err != nil { + return "", "", err + } + return issued, authority.Certificate, nil +} + +// grantsFor is every credential this node must create, because something elsewhere uses it. +// +// The mirror of what a consumer is given, and the half that makes the credential real: a password +// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so +// the mesh hands over something it cannot itself use. +func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.Grant, error) { + issued, err := inv.SecretsFrom(ctx, node) + if err != nil { + return nil, err + } + + // Where each consumer is, so a provider that must reach back to one does not have to know how + // the mesh names machines. + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + onNetwork, err := whereEveryoneIs(ctx, inv, shelf) + if err != nil { + return nil, err + } + + // What each consumer actually asked for, taken from that machine's own resolution rather than + // from a record beside it. A provider told to create a password and not what to create it for + // can do nothing with it, and the name a consumer wants is the consumer's to say. + out := make([]catalogue.Grant, 0, len(issued)) + for _, s := range issued { + plan, settings, err := planFor(ctx, inv, s.Consumer) + if err != nil { + // Their set does not resolve. Skipped rather than fatal: this node is not the place + // to report another machine's problem, and a grant for something that is not going to + // run would have the provider create a user nothing uses. + continue + } + from, values, err := plan.ContributionsTo(s.Name, settings) + if err != nil { + return nil, err + } + out = append(out, catalogue.Grant{ + Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer], + From: from, Values: values, Sealed: s.ForProvider}) + } + return out, nil +} + +func planCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("plan", flag.ContinueOnError) + // Because "one resource" does not tell you whether the settings landed. Being able to read + // the file before it is sent is the difference between believing a merge worked and knowing. + show := set.Bool("files", false, "print the files this node would be given") + // The declaration exactly as the node would receive it. For handing to something else -- + // checking it against the host's own parser, most usefully, which is the only way to know + // that what the control plane emits is what the host accepts. + asJSON := set.Bool("json", false, "print the declaration this node would be sent") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("plan [--files] [--json]") + } + args = positionals + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + plan, settings, err := planFor(ctx, inv, args[0]) + if err != nil { + return err + } + if len(plan.Modules) == 0 { + fmt.Printf("%s is assigned nothing\n", args[0]) + return nil + } + if *asJSON { + resources, err := declarationFor(ctx, inv, args[0], plan, settings) + if err != nil { + return err + } + body, err := json.MarshalIndent( + map[string]any{"declaration": 1, "resources": resources}, "", " ") + if err != nil { + return err + } + fmt.Println(string(body)) + return nil + } + + fmt.Printf("%s would run:\n", args[0]) + for _, m := range plan.Modules { + fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module]) + } + for _, c := range plan.Claims { + fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope) + } + // What this machine depends on that is not on it. Worth saying out loud: it is the only part + // of a node's set that stops working when a *different* machine goes away, and nothing else + // in this output would have told anybody that. + for _, n := range plan.Needs { + fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) + } + resources, err := declarationFor(ctx, inv, args[0], plan, settings) + if err != nil { + return err + } + for module, layers := range settings { + for _, layer := range layers { + fmt.Printf(" %-20s settings from %s\n", module, layer.From) + } + } + fmt.Printf("\n%d resource(s)\n", len(resources)) + + if *show { + for _, r := range resources { + content, ok := r["content"].(string) + if !ok { + continue + } + fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content) + } + } + return nil +} + +// licencesFor is what this node can be answered with by record, and what it was put on. +// +// A mesh with no licences at all is the ordinary case and must not be an error: every existing +// mesh is one, and a control plane that refused to plan because nobody had bought an API key +// would be unusable for the thing it already does. +func licencesFor(ctx context.Context, node string) ( + map[string][]catalogue.Record, map[string]map[string]catalogue.Record, error) { + + held, err := openLicences(ctx) + if err != nil { + return nil, nil, err + } + defer held.Close() + + all, err := held.All(ctx) + if err != nil { + return nil, nil, err + } + if len(all) == 0 { + return nil, nil, nil + } + + offered := map[string][]catalogue.Record{} + byName := map[string]catalogue.Record{} + for _, one := range all { + record := catalogue.Record{Name: one.Name, Serves: one.Serves} + offered[licences.Provision] = append(offered[licences.Provision], record) + byName[one.Name] = record + } + + using := map[string]map[string]catalogue.Record{} + for _, one := range all { + holders, err := held.HoldersOf(ctx, one.Name) + if err != nil { + return nil, nil, err + } + for _, h := range holders { + if h.Node != node { + continue + } + if using[h.Module] == nil { + using[h.Module] = map[string]catalogue.Record{} + } + using[h.Module][licences.Provision] = byName[one.Name] + } + } + return offered, using, nil +} + +// keyFor is the licence key sealed to one machine, for one module. +// +// **Empty is not an error here.** The mesh discarded the plaintext when it was supplied, so a +// holder recorded afterwards genuinely has no key — and the declaration refuses that by name, +// where the module and the path are both in view, rather than here. +func keyFor(ctx context.Context, licence, node, module string) (string, error) { + held, err := openLicences(ctx) + if err != nil { + return "", err + } + defer held.Close() + return held.KeyFor(ctx, licence, node, module) +} diff --git a/cmd/mesh-control/push.go b/cmd/mesh-control/push.go new file mode 100644 index 0000000..e03fe01 --- /dev/null +++ b/cmd/mesh-control/push.go @@ -0,0 +1,409 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-control/internal/broker" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/link" +) + +// sending it, and holding the link that carries it. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// serve is the control plane running: one connection to the broker, one queue, one consumer. +func serve(ctx context.Context) error { + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + // Established at start rather than on first use. A control plane that cannot sign is one + // whose declarations every node correctly refuses, and that should be a startup failure + // rather than something discovered at the first declaration. + key, err := ident.Establish(ctx) + if err != nil { + return err + } + fmt.Printf("signing as %s\n", key.Fingerprint()[:16]) + + management, err := broker.ManagementFromEnvironment() + if err != nil && !errors.Is(err, broker.ErrNotConfigured) { + return err + } + + // Where the broker is and what to expect there, so a node can be told how to come back + // without a person and a new token. + known, err := broker.FromEnvironment() + if err != nil && !errors.Is(err, broker.ErrNotConfigured) { + return err + } + if errors.Is(err, broker.ErrNotConfigured) { + fmt.Printf("no broker address configured, so enrolled nodes will not be told how to "+ + "reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar) + } + + work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known} + server, err := link.Connect(work, work) + if err != nil { + return err + } + defer server.Close() + // And build results nobody was waiting for. A build triggered any other way than `build` + // would otherwise be reported into the void, which is the same as not reporting it. + server.Records(builds{inv}) + + return server.Serve(ctx) +} + +// declare sends one node a declaration, signed. +// +// Signed here rather than trusted from the broker: a node connects to the broker and takes +// instruction from the control plane behind it, and those are two identities. If a node believed +// whatever arrived on its queue, a compromised broker could forge declarations — and since the +// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004). +func declare(ctx context.Context, args []string) error { + if len(args) != 2 { + return errors.New("declare ") + } + node, path := args[0], args[1] + + raw, err := os.ReadFile(path) + if err != nil { + return err + } + + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + // The node has to exist before it can be told anything. Publishing to a queue nobody consumes + // would sit there looking like success. + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + if _, err := inv.NodeByName(ctx, node); err != nil { + return err + } + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil { + return err + } + fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) + return nil +} + +// OverlayCIDRVar is the range the mesh allocates node addresses from. +const OverlayCIDRVar = "MESH_OVERLAY_CIDR" + +// pushCommand sends nodes everything they should be: their place on the network, and what their +// assignments resolve to. +// +// One declaration, not two. A node holding its network and not its modules, or the reverse, is +// half-configured for as long as that lasts — and the two are computed from the same picture of +// the mesh, so sending them apart would let them disagree. +func pushCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("push", flag.ContinueOnError) + // Only the machines that need it. + // + // **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is + // a scheduler over this, and building the scheduler first would mean two paths to one act + // with nothing to compare them against. A person can run this; so can cron; so can whatever + // eventually watches. + behind := set.Bool("behind", false, + "only machines whose last declaration was refused or partly failed") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + args = positionals + if len(args) > 1 { + return errors.New("push [] [--behind] — one node, or all of them") + } + if len(args) == 1 && *behind { + // Naming a machine and asking for the ones that need it are two different requests, and + // guessing which was meant would sometimes push to a machine somebody did not name. + return errors.New("push or push --behind, not both: one names a machine and the " + + "other asks which machines need one") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + // Every node, not only the ones on the private network. A machine that was never given the + // network module still takes modules, and iterating the network here is what used to make + // "on the network" and "managed" the same thing. + nodes, err := inv.Nodes(ctx) + if err != nil { + return err + } + + // Which machines are not in the state they were sent, when that is what was asked for. + var needsOne map[string]inventory.Doing + if *behind { + wrong, err := inv.NotDoingWhatTheyWereTold(ctx) + if err != nil { + return err + } + needsOne = map[string]inventory.Doing{} + for _, d := range wrong { + needsOne[d.Node] = d + } + // **And every machine not running what the mesh would send it.** "Behind" used to mean + // only "failed or refused", so a machine that applied cleanly and whose declaration has + // since changed was not behind — and novox/hq ADR 0010's question, *did my change go + // out?*, was answerable only for the machines that broke. + would, err := wouldSend(ctx, inv, nodes) + if err != nil { + return err + } + waiting, err := inv.Waiting(ctx, would) + if err != nil { + return err + } + for _, m := range waiting { + if _, already := needsOne[m.Node]; already { + continue + } + needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} + } + if len(needsOne) == 0 { + // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" + // must never look the same. + fmt.Println("every machine is doing what it was told") + return nil + } + } + gens, err := generators(ctx, inv) + if err != nil { + return err + } + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + // Every node is resolved before anything is sent. A push that configured three nodes and then + // refused on the fourth would leave the mesh in a state nobody asked for, and the fourth is + // exactly where a claim collision shows up. + type ready struct { + node string + resources []map[string]any + } + var sending []ready + var refusals []string + + for _, n := range nodes { + if len(args) == 1 && n.Name != args[0] { + continue + } + if *behind { + doing, needs := needsOne[n.Name] + if !needs { + continue + } + // A machine that has been failing the same way for a long time is not going to stop + // because it was asked again. Said, and pushed to anyway — refusing would leave no + // way to retry after fixing the cause, and this is a command somebody ran. + // + // Only for machines that reported something. One that is merely waiting has no report + // to be old, and saying it had been failing since the zero time would be a sentence + // about nothing. + if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { + fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ + "unlikely to be timing\n", + n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) + } + } + plan, settings, err := planFor(ctx, inv, n.Name) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err)) + continue + } + // The private network is in here with everything else. It used to be composed separately + // and prepended, which meant every machine with an address was on it and no machine could + // be kept off. It is a module now, so it arrives the way a module does. + resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err)) + continue + } + if len(resources) == 0 { + fmt.Printf("%s is assigned nothing — skipped\n", n.Name) + continue + } + sending = append(sending, ready{n.Name, resources}) + } + + if len(refusals) > 0 { + return fmt.Errorf("nothing was sent. %d node(s) could not be resolved:\n\n%s", + len(refusals), strings.Join(refusals, "\n\n")) + } + + for _, s := range sending { + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + if err != nil { + return err + } + if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + return err + } + // After it is away, not before. A digest recorded for something that failed to send would + // make the machine look current for a declaration it never received. + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } + fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources)) + } + fmt.Printf("\n%d node(s) told\n", len(sending)) + return nil +} + +// sendTo resolves and sends to exactly the machines named, or refuses without sending anything. +// +// The same all-or-nothing rule push follows, and for the same reason: a rotation that reached the +// consumer and refused on the provider would leave one end holding a credential the other has +// never heard of — which is the state this whole mechanism exists to make impossible. +func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error { + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + gens, err := generators(ctx, inv) + if err != nil { + return err + } + + type ready struct { + node string + resources []map[string]any + } + var sending []ready + var refusals []string + for _, name := range names { + plan, settings, err := planFor(ctx, inv, name) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) + continue + } + resources, err := declarationWith(ctx, inv, name, plan, settings, gens) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) + continue + } + sending = append(sending, ready{name, resources}) + } + if len(refusals) > 0 { + return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", + len(refusals), strings.Join(refusals, "\n\n")) + } + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + for _, s := range sending { + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + if err != nil { + return err + } + if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + return err + } + record, err := inv.NodeByName(ctx, s.node) + if err != nil { + return err + } + if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil { + return err + } + fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) + } + return nil +} + +// digestOf is what the mesh compares to answer "has this machine been sent what it should be". +// +// Over the same bytes that are sent, so the comparison is of the thing itself rather than of +// something derived beside it that could drift from it. +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +// wouldSend is the digest of what each machine should be right now. +// +// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot +// be worked out" is a different problem with a different remedy, and `plan` is where it is said. +func wouldSend(ctx context.Context, inv *inventory.Inventory, + nodes []inventory.Node) (map[string]string, error) { + + gens, err := generators(ctx, inv) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, n := range nodes { + plan, settings, err := planFor(ctx, inv, n.Name) + if err != nil { + continue + } + resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) + if err != nil { + continue + } + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources}) + if err != nil { + return nil, err + } + out[n.Name] = digestOf(body) + } + return out, nil +} diff --git a/cmd/mesh-control/status.go b/cmd/mesh-control/status.go new file mode 100644 index 0000000..69978b3 --- /dev/null +++ b/cmd/mesh-control/status.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "flag" + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-control/internal/inventory" +) + +// is anything broken, is anything not answering, is anything out of date. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// short is a commit as a person refers to it. +func short(commit string) string { + if len(commit) > 8 { + return commit[:8] + } + return commit +} + +// statusCommand answers "did my change go out?". +// +// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a +// comparison: it is answerable today by opening a pipeline, and something has to replace that or +// this is worse to live with whatever its other properties. +// +// The answer is not "a job succeeded". It is which modules the mesh has not built from what their +// source now has, and which machines are running the old one. +func statusCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("status", flag.ContinueOnError) + asJSON := set.Bool("json", false, "the same answers, for something other than a person") + if _, err := parseAround(set, args); err != nil { + return err + } + + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + asked, err := theThreeQuestions(ctx, inv) + if err != nil { + return err + } + wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet + behind, sources := asked.behind, asked.sources + + if *asJSON { + body, err := statusAsJSON(wrong, nodes, quiet, behind, sources, asked.waiting) + if err != nil { + return err + } + fmt.Println(string(body)) + return nil + } + + if len(wrong) > 0 { + fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong)) + for _, d := range wrong { + fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04")) + if d.Refused != "" { + // The host's own words. It says exactly what it could not accept, and nothing + // written here would say it better. + fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused)) + } + for _, f := range d.Failed { + fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error)) + } + } + fmt.Println() + } + + if len(quiet) > 0 { + var said []string + for _, n := range quiet { + said = append(said, n.Name+" ("+heardFrom(n)+")") + } + fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n", + len(quiet), strings.Join(said, "\n ")) + } + + if len(behind) > 0 { + var names []string + for m := range behind { + names = append(names, m) + } + sort.Strings(names) + + fmt.Printf("%d module(s) behind their source:\n\n", len(behind)) + for _, m := range names { + from := sources[m] + fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head)) + if on := behind[m]; len(on) > 0 { + // The part somebody actually wants. A module being out of date is a fact about + // the catalogue; machines running the old one is the thing with consequences. + fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", ")) + } else { + fmt.Printf(" %-18s assigned to nothing\n", "") + } + } + // The remedy, beside the problem. A status that says what is wrong and not what to do + // about it makes somebody go and find the command, and the command is the whole point of + // having noticed. + fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n") + fmt.Println() + } + + if len(asked.waiting) > 0 { + // The other half of "is anything out of date": a module behind its source says the + // catalogue is old, and this says a machine is — and only this one has somebody's change + // waiting inside it. + var told, never []string + for _, m := range asked.waiting { + if m.Never { + never = append(never, m.Node) + continue + } + told = append(told, m.Node) + } + if len(told) > 0 { + fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n", + len(told), strings.Join(told, ", ")) + } + if len(never) > 0 { + // Never told is not out of date. The remedy is the same push and the situation is + // not the same at all: nobody has ever asked this machine to be anything. + fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", + len(never), strings.Join(never, ", ")) + } + fmt.Printf("\n `push --behind` sends them\n\n") + } + + if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 { + // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, + // and getting here means every question was asked and answered. + fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+ + "the mesh would send them, and every module current with its source\n", len(nodes)) + } + return nil +} + +// firstLine is as much of a failure as belongs in a list. +func firstLine(s string) string { + if cut := strings.IndexByte(s, '\n'); cut >= 0 { + return strings.TrimSpace(s[:cut]) + } + return strings.TrimSpace(s) +} + +// builds keeps what a builder said, for the serving control plane. +// +// A type of its own rather than a method on the enrolment, because they are unrelated things +// arriving on one queue and an implementation of one should not have to say anything about the +// other. +type builds struct{ inv *inventory.Inventory } + +// theThreeQuestions reads what anything answering "is the mesh alright" needs. +// +// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There +// are three now — a person's status, its JSON, and a page — and three implementations of "which +// machine is not doing what it was told" would be three chances to disagree about it. +// +// The order is the design and not a convenience: is anything broken, is anything not answering, is +// anything out of date. The first has consequences now, the second may, the third is a plan for +// later — and anything that led with the third would bury the first. +func theThreeQuestions(ctx context.Context, inv *inventory.Inventory) (answers, error) { + var out answers + var err error + + out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx) + if err != nil { + return answers{}, err + } + out.nodes, err = inv.Nodes(ctx) + if err != nil { + return answers{}, err + } + for _, n := range out.nodes { + // Never heard from, or not lately. Different from failing: a machine that says nothing + // may be new, switched off, or unreachable, and none of those is a machine that tried + // and could not. + if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour { + out.quiet = append(out.quiet, n) + } + } + out.behind, err = inv.Behind(ctx) + if err != nil { + return answers{}, err + } + // And which machines are not running what the mesh would send them. The same question as a + // module being behind its source, one level down: that one says the catalogue is out of date, + // this one says a machine is — and only the second has anybody's change waiting in it. + would, err := wouldSend(ctx, inv, out.nodes) + if err != nil { + return answers{}, err + } + out.waiting, err = inv.Waiting(ctx, would) + if err != nil { + return answers{}, err + } + out.sources = map[string]inventory.Source{} + for module := range out.behind { + from, err := inv.SourceOf(ctx, module) + if err != nil { + return answers{}, err + } + out.sources[module] = from + } + return out, nil +} diff --git a/cmd/mesh-control/stores.go b/cmd/mesh-control/stores.go new file mode 100644 index 0000000..69263f6 --- /dev/null +++ b/cmd/mesh-control/stores.go @@ -0,0 +1,113 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/novox/mesh-control/internal/identity" + "github.com/novox/mesh-control/internal/inventory" + "github.com/novox/mesh-control/internal/licences" + "github.com/novox/mesh-control/internal/store" +) + +// reaching each context's store, which no other context may touch. +// +// Split out of main.go, which had reached 2,769 lines because appending was always the +// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: +// nothing in it was wrong, and no one edit was the one that should have been a new file. + +// migrate brings every held context's schema up to date. +// +// Reported per context and per migration, because this runs during a bootstrap on a machine with +// nothing else on it — the output is the only account of what happened, and "migrated" is not one. +func migrate(ctx context.Context) error { + for _, c := range held { + migrations, err := c.migrations() + if err != nil { + return err + } + + s, err := store.Open(ctx, c.name) + if err != nil { + return err + } + defer s.Close() + + // The bootstrap raises PostgreSQL moments before this runs, and a container that is + // running is not a database that will answer — a distinction this project has already + // paid for once, when a crash-looping database reported itself as up between restarts. + if err := s.Ready(ctx, 60*time.Second); err != nil { + return err + } + + done, err := s.Migrate(ctx, migrations) + for _, m := range done { + fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name) + } + if err != nil { + return err + } + if len(done) == 0 { + applied, err := s.AppliedMigrations(ctx) + if err != nil { + return err + } + fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied)) + } + } + + // The modules the control plane ships with itself. Recorded here rather than by hand, because + // a mesh whose own private network is missing from the catalogue would have nothing to assign + // and no way to say why. + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + for _, m := range provided { + if err := inv.Provide(ctx, m); err != nil { + return err + } + fmt.Printf("provided %s\n", m.Module) + } + return nil +} + +// openInventory connects and waits, the way every command that touches it needs to. +func openInventory(ctx context.Context) (*inventory.Inventory, error) { + inv, err := inventory.Open(ctx) + if err != nil { + return nil, err + } + if err := inv.Ready(ctx, 30*time.Second); err != nil { + inv.Close() + return nil, err + } + return inv, nil +} + +func openIdentity(ctx context.Context) (*identity.Identity, error) { + ident, err := identity.Open(ctx) + if err != nil { + return nil, err + } + if err := ident.Ready(ctx, 30*time.Second); err != nil { + ident.Close() + return nil, err + } + return ident, nil +} + +// openLicences connects to the context that holds which model access exists and who may use it. +func openLicences(ctx context.Context) (*licences.Licences, error) { + held, err := licences.Open(ctx) + if err != nil { + return nil, err + } + if err := held.Ready(ctx, 30*time.Second); err != nil { + held.Close() + return nil, err + } + return held, nil +}