jschoubben
e419d0182f
The controller asks no build of anybody (ADR 0501 §5, ADR 0525 clause 6, ADR 0503 §3). Every module of a walk is taken from the builder's done key; a key not done yet waits, named in the plan, and never fails it. - C2b/C2c: internal/builder and cmd/mesh-controller's asks go (askABuild, onlyTheBuildersOwn, buildOneAsked, takeIn from an outcome, buildAgentsOwn, the builder's say and its fallback, the daemon hearing an outcome). The server follows the retired seats' built subjects only to let go of them, because the installer's carried user list names them. - C2d: replay, queue, cancel, clear, kill, pause, resume, delivery-check, check-here and merge-gate are refused by name. - C3: the check queues (CHECKS_READY, CHECKS_OTHER) and their workers, the checks-running and plans-building buckets and their grants, the build queue and its hold (plans_building.go, queue.go, link/holds.go, builds_nats.go, checks_nats.go). - C4: merge_gate*.go, the pull request's repo-check path, the Checked event and its layers; a pull request asks the controller nothing but to tell mesh-config what its head declares. - C5: the builds bucket is written only by key_takein. - C10 (help and refusals): the terminal's help lists only what is served. Integration tests of the plan's tiers, sends, stops and retries now run on a stand-in builder with a done key for every module (walk_keys_scaffold_integration_test.go); the tests of the asks, the queue, the merge gate and the replays are deleted.
415 lines
21 KiB
Go
415 lines
21 KiB
Go
// Command mesh-controller is the control plane: everything that needs to know about more than one
|
|
// node (novox/hq ADR 0006).
|
|
//
|
|
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
|
|
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
|
|
// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/identity"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/licences"
|
|
"github.com/novox/mesh-controller/internal/store"
|
|
)
|
|
|
|
// version is stamped at link time. Unset in a development build, and it says so rather than
|
|
// claiming a number.
|
|
var version = "development build"
|
|
|
|
// held is a context this process was granted, and the schema it carries.
|
|
//
|
|
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
|
|
// yet, not because they are optional.
|
|
var held = []struct {
|
|
name string
|
|
migrations func() ([]store.Migration, error)
|
|
}{
|
|
{inventory.Name, inventory.Migrations},
|
|
{identity.Name, identity.Migrations},
|
|
{licences.Name, licences.Migrations},
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
args := os.Args[1:]
|
|
if len(args) == 0 {
|
|
usage()
|
|
return fmt.Errorf("no command given")
|
|
}
|
|
|
|
// The bus's users are composed from what each module announced it keeps (novox/hq ADR 0495, issue 1790).
|
|
inventory.Announcements = readAnnouncements
|
|
// Every connection this process dials says what it is (novox/hq issue 327).
|
|
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
|
// 45 §6), so the next controller takes it at once rather than after its age.
|
|
defer theLease.release()
|
|
|
|
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
|
|
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
|
|
// own saves.
|
|
if args[0] != "serve" {
|
|
said, done := sayingWhatAVerbKept(ctx)
|
|
inventory.PlanSaved = said
|
|
defer done()
|
|
}
|
|
|
|
switch args[0] {
|
|
case "switch":
|
|
return switchCommand(ctx, args[1:])
|
|
case "register":
|
|
return registerCommand(ctx, args[1:])
|
|
case "build", "rebuild":
|
|
// **The hand build goes with the asks** (novox/hq ADR 0501 §5): refused by name, naming the replacement.
|
|
return removedByName(args[0])
|
|
case "board":
|
|
return boardCommand(ctx, args[1:])
|
|
case "api":
|
|
return apiCommand(ctx, args[1:])
|
|
case "licence":
|
|
return licenceCommand(ctx, args[1:])
|
|
case "rotate":
|
|
return rotateCommand(ctx, args[1:])
|
|
case "ask":
|
|
return askCommand(ctx, args[1:])
|
|
case "rehearse":
|
|
return rehearseCommand(ctx, args[1:])
|
|
case "builds":
|
|
return buildsCommand(ctx, args[1:])
|
|
case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume":
|
|
return removedByName(args[0])
|
|
case "try":
|
|
return tryCommand(ctx, args[1:])
|
|
case "collection":
|
|
return collectionCommand(ctx, args[1:])
|
|
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251).
|
|
case "artifacts":
|
|
return artifactsCommand(ctx, args[1:])
|
|
case "collect":
|
|
return collectCommand(ctx, args[1:])
|
|
case "images":
|
|
return imagesCommand(ctx, args[1:])
|
|
case "mirrors":
|
|
return mirrorsCommand(ctx, args[1:])
|
|
case "move-placement":
|
|
return movePlacementCommand(ctx, args[1:])
|
|
case "move-config":
|
|
return moveConfigCommand(ctx, args[1:])
|
|
case "plans":
|
|
return plansCommand(ctx, args[1:])
|
|
case "delivery":
|
|
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
|
|
return deliveryCommand(ctx, args[1:])
|
|
case "pin":
|
|
return pinCommand(ctx, args[1:], true)
|
|
case "unpin":
|
|
return pinCommand(ctx, args[1:], false)
|
|
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
|
|
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
|
|
// own schema is not a special case. `migrate` remains the word a person types.
|
|
case "prepare", "migrate":
|
|
return migrate(ctx)
|
|
case "node":
|
|
return nodeCommand(ctx, args[1:])
|
|
case "token":
|
|
return tokenCommand(ctx, args[1:])
|
|
case "identity":
|
|
return identityCommand(ctx, args[1:])
|
|
case "broker":
|
|
return brokerCommand(ctx, args[1:])
|
|
case "serve":
|
|
return serve(ctx)
|
|
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
|
|
case "facts":
|
|
return factsCommand(ctx, args[1:])
|
|
case "upgrade":
|
|
return upgradeCommand(ctx, args[1:])
|
|
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
|
case "bus":
|
|
return busCommand(ctx, args[1:])
|
|
case "declare":
|
|
return declare(ctx, args[1:])
|
|
case "overlay":
|
|
return overlayCommand(ctx, args[1:])
|
|
case "module":
|
|
return moduleCommand(ctx, args[1:])
|
|
case "assign", "unassign":
|
|
return assignCommand(ctx, args[0], args[1:])
|
|
case "take":
|
|
return takeCommand(ctx, args[1:])
|
|
case "converge":
|
|
return convergeCommand(ctx, args[1:])
|
|
case "adopt":
|
|
return adoptCommand(ctx, args[1:])
|
|
case "settings":
|
|
return settingsCommand(ctx, args[1:])
|
|
case "secret":
|
|
return secretCommand(ctx, args[1:])
|
|
case "operator":
|
|
return operatorCommand(ctx, args[1:])
|
|
case "plan":
|
|
return planCommand(ctx, args[1:])
|
|
case "push":
|
|
return pushCommand(ctx, args[1:])
|
|
case "rollout":
|
|
return rolloutCommand(ctx, args[1:])
|
|
case "seats":
|
|
return seatsCommand(ctx, args[1:])
|
|
case "seat":
|
|
return seatCommand(ctx, args[1:])
|
|
case "status":
|
|
return statusCommand(ctx, args[1:])
|
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
|
case "hand-act":
|
|
return handActCommand(ctx, args[1:])
|
|
case "hand-acts":
|
|
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
|
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
|
case "durations":
|
|
return durationsCommand(ctx, args[1:])
|
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
|
case "conditions":
|
|
return conditionsCommand(ctx, args[1:])
|
|
case "doctor":
|
|
return doctorCommand(ctx, args[1:])
|
|
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
|
case "healers":
|
|
return healersCommand(ctx, args[1:])
|
|
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
|
// (novox/hq ADR 0230).
|
|
case "retire":
|
|
return retireCommand(ctx, args[1:])
|
|
case "cleanup":
|
|
return cleanupCommand(ctx, args[1:])
|
|
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
|
case "data":
|
|
return dataCommand(ctx, args[1:])
|
|
case "version":
|
|
fmt.Println(version)
|
|
return nil
|
|
case "help", "-h", "--help":
|
|
usage()
|
|
return nil
|
|
default:
|
|
usage()
|
|
return fmt.Errorf("%q is not a command", args[0])
|
|
}
|
|
}
|
|
|
|
// usageText is the terminal's help: every command it has, and none it refuses as removed (removedByName).
|
|
var usageText = `mesh-controller — the control plane
|
|
|
|
On a node the operator types these as 'mesh-cli <command>' (in zsh, 'nox <command>'): asked
|
|
through the node's engine, and run as the controller's terminal only on the control-node
|
|
(novox/hq ADR 0272).
|
|
|
|
migrate bring each context's schema up to date
|
|
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
|
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
|
node list the nodes this mesh knows about
|
|
node show <name> what one machine reported it can do, and why
|
|
node public-domain <name> the domain it composes its routed names under
|
|
node public-domain <name> <d> ...set it to d
|
|
node public-domain <name> --clear ...it faces the outside no longer
|
|
node networks <name> the networks it routes for what it hosts
|
|
node networks <name> <cidr>... ...set them; its filter forwards these too
|
|
node networks <name> --clear ...only the container runtime's own
|
|
token issue --node <name> a one-time right to join, for an existing record
|
|
token issue --new <name> create the record and issue for it
|
|
token issue ... --adopted ...for a machine in use, which joins adopted
|
|
identity show this control plane's signing key
|
|
broker show where the broker is, and what to expect there
|
|
serve consume what nodes say, and answer
|
|
declare <node> <file> send a node a signed declaration
|
|
overlay place <node> [flags] say where a node is and how it is reached
|
|
overlay show the private network, as the mesh computes it
|
|
module add <file> register a module from its manifest
|
|
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
|
module list what modules this mesh knows about
|
|
module moved <name> <commit> the source has a newer commit than the mesh built
|
|
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
|
module forget <name> --and-what-it-holds ...and discard its settings, secrets and ports too
|
|
module issue <name> --node <m> a broker account for a module, scoped to its emits and consumes
|
|
upgrade <name> what happens when this module's current version moves
|
|
upgrade <name> roll-out [--together] ...send it to the machines running it
|
|
upgrade <name> record ...record that they are behind, and send nothing
|
|
status [--json] what is wrong, what is quiet, and what is out of date
|
|
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
|
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
|
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
|
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
|
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
|
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
|
artifacts [--json] [--repository <r>] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251)
|
|
collect [--json] [--most <n>] what the store's sweep would let go of, now; nothing is done
|
|
collect --confirm --why <text> [--most <n>] run the sweep now: hold every kept archive, let go of the eligible
|
|
move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved
|
|
move-placement --confirm --why <text> move them beside the assignment, once, after the delivery (issue 736)
|
|
move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set
|
|
move-config --confirm --why <text> set them in mesh-config, read them back, and read config from there (issue 664)
|
|
images <node> [--json] every container image the machine's declaration names, now and as last sent
|
|
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
|
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
|
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
|
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
|
board [--listen ADDR] the same three questions, as a page that holds nothing
|
|
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
|
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
|
unassign <node> <module>... take them off
|
|
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
|
what it declares; --yes <digest> cuts it over as previewed
|
|
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
|
adopt <node> return a converged node to adopted; what was taken stays taken
|
|
settings set <module> <file> what a module's config should say, for the whole mesh
|
|
settings set <module> <file> --node <n> ...or for one machine
|
|
settings clear <module> [--node <n>] take a layer away
|
|
secret accept <node> <module> <name> carry a value the mesh did not make and cannot invent
|
|
secret accept ... --from <file> ...read it from a file rather than being asked
|
|
secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
|
|
break-glass: open the operator-sealed copy, to a 0600 file
|
|
secret export [--out <file>] every operator-sealed copy, ciphertext — keep it with the key
|
|
operator key make [--out <file>] make the operator's sealing key, off the mesh; private half to the file only
|
|
operator key set <public> [--replace] tell the mesh which operator key to seal to
|
|
operator key show the operator key, and what it can recover
|
|
register <module> <owner>/<repository> <path> register a module from a repository the catalogue does not
|
|
build, from the builder's done key of its fingerprint (ADR 0501 §4); with
|
|
--reference <artifact>=<reference> (repeatable) and --why, at artifacts the store
|
|
holds, recorded as a build by hand (ADR 0526 §3)
|
|
switch [--json] the switch's list: each module, the build it runs, and what the builder holds
|
|
builds [<module>] [-n N] what has been built lately: the builder's done keys, recorded as builds
|
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
|
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
|
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
|
record an act done by hand outside the mesh (to-be 45 §7)
|
|
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
|
conditions [--scope S] [--severity S] [--machine M] [--json]
|
|
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
|
conditions show <key> one condition whole, with its evidence
|
|
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
|
conditions history [--days N] [--key K] every raising, change and clearing lately
|
|
doctor [run|probes|signals] [--json]
|
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
|
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
|
durations [--kind K] [--days N] [--json]
|
|
apply, heartbeat, plan-tier, build and walk-phase durations
|
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
|
licence add|list|use|key model access, under the name a person calls it
|
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
|
licence refresh <name> mint a new access token and seal it to every holder
|
|
rotate <provision> [--consumer <n>] [--module <m>] [--why <text>] a new credential for every holder, both ends at once (--why where the vault holds it)
|
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
|
pin <node> <provision> <from-node> <module>
|
|
which provider this one gets a provision from: the module, and its node
|
|
unpin <node> <provision> put that question back
|
|
plan <node> [--files|--json] what that node would run, and why
|
|
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
|
that need it; --why records it in the hand-act log
|
|
version what this binary is
|
|
|
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
|
` + store.Variable("<context>") + ` — or ` + store.FileVariable("<context>") + `, naming a file that holds
|
|
the same thing and keeps the password out of the environment. This process holds:
|
|
|
|
`
|
|
|
|
func usage() {
|
|
fmt.Fprint(os.Stderr, usageText)
|
|
for _, c := range held {
|
|
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
|
c.name, store.Database(c.name), store.Variable(c.name))
|
|
}
|
|
fmt.Fprintln(os.Stderr)
|
|
}
|
|
|
|
// parseAround reads flags that may sit before, after or between positional arguments.
|
|
//
|
|
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
|
|
// parses no flags at all and silently ignores every one of them. The host learned this the same
|
|
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
|
|
// keeps naming, and it looks exactly like success.
|
|
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
|
var positionals []string
|
|
rest := args
|
|
for {
|
|
if err := set.Parse(rest); err != nil {
|
|
return nil, err
|
|
}
|
|
rest = set.Args()
|
|
if len(rest) == 0 {
|
|
return positionals, nil
|
|
}
|
|
positionals = append(positionals, rest[0])
|
|
rest = rest[1:]
|
|
}
|
|
}
|
|
|
|
// verbVar carries the seat verb a command runs for, from the serving controller to the process it starts.
|
|
const verbVar = "MESH_VERB"
|
|
|
|
// connectionName is what this process's connections say they are in the bus's list (novox/hq issue 327):
|
|
// the serving controller, a verb's own process and which verb, or a command run at a shell and which.
|
|
func connectionName(command, verb string) string {
|
|
switch {
|
|
case command == "serve":
|
|
return "mesh-controller serving"
|
|
case verb != "":
|
|
return "mesh-controller verb " + verb
|
|
}
|
|
return "mesh-controller command " + command
|
|
}
|
|
|
|
// removedByName refuses a command the controller no longer has, naming what replaced it (novox/hq ADR 0501 §4, §5;
|
|
// ADR 0429 §18; ADR 0503 §3; issue 1851, A11, and issue 1318).
|
|
func removedByName(command string) error {
|
|
replacement := map[string]string{
|
|
"build": "the controller asks no build (ADR 0501 §5): the builder builds every fingerprint the indexer writes, a " +
|
|
"module is registered from its done key, and a branch is tried with `try`; a module from a repository the " +
|
|
"catalogue does not build is registered at the controller's terminal with `register`",
|
|
"rebuild": "the controller asks no build (ADR 0501 §5): a failed key is built again by the builder as a new " +
|
|
"revision (ADR 0429 §18), and a walk's Retry waits for it",
|
|
"replay": "the controller asks no build, so there is nothing to replay (ADR 0501 §5, ADR 0525 clause 6); a branch is " +
|
|
"tried with `try`",
|
|
"queue": "the controller keeps no build queue (ADR 0525 clause 6): the builds and checks are asks on the build modules' " +
|
|
"streams, and the delivery view shows their backlog and the oldest ask's age",
|
|
"cancel": "the controller keeps no build queue (ADR 0525 clause 6): an ask is on a build module's stream, and the delivery " +
|
|
"view shows the backlog",
|
|
"clear": "the controller keeps no build queue (ADR 0525 clause 6): an ask is on a build module's stream, and the delivery " +
|
|
"view shows the backlog",
|
|
"kill": "the controller keeps no build queue and starts no build (ADR 0525 clause 6): a build runs on a build module, " +
|
|
"which ends it",
|
|
"pause": "the build seat is retired (ADR 0525 clause 7): a node's builds are paused with its build module's own pause, " +
|
|
"or a capacity of 0",
|
|
"resume": "the build seat is retired (ADR 0525 clause 7): a node's builds are resumed with its build module's own " +
|
|
"resume, or its capacity",
|
|
"forget-checks": "build-agent makes no check build from R4e (ADR 0503 §3), so there is nothing to let go of",
|
|
"delivery-plan": "a delivery's plan is read with `delivery-order` and the checker's verdict (ADR 0503 §3)",
|
|
"delivery-check": "the checker judges every head and every delivery group (ADR 0503 §3, ADR 0527): the controller asks " +
|
|
"nothing and says no verdict of a pull request",
|
|
}[command]
|
|
return &removedCommand{msg: fmt.Sprintf("`%s` is removed: %s", command, replacement)}
|
|
}
|
|
|
|
// removedCommand is a command or verb the controller no longer has: a refusal that is the verb's whole answer, not a
|
|
// verb of a newer build this one cannot run.
|
|
type removedCommand struct{ msg string }
|
|
|
|
func (r *removedCommand) Error() string { return r.msg }
|