jochen
52af210e47
A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten.
376 lines
18 KiB
Go
376 lines
18 KiB
Go
// Command mesh-controller is the control plane: everything that needs to know about more than one
|
|
// node (novox/hq ADR 0006).
|
|
//
|
|
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
|
|
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
|
|
// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"github.com/novox/mesh-controller/internal/identity"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/licences"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/store"
|
|
)
|
|
|
|
// version is stamped at link time. Unset in a development build, and it says so rather than
|
|
// claiming a number.
|
|
var version = "development build"
|
|
|
|
// held is a context this process was granted, and the schema it carries.
|
|
//
|
|
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
|
|
// yet, not because they are optional.
|
|
var held = []struct {
|
|
name string
|
|
migrations func() ([]store.Migration, error)
|
|
}{
|
|
{inventory.Name, inventory.Migrations},
|
|
{identity.Name, identity.Migrations},
|
|
{licences.Name, licences.Migrations},
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
args := os.Args[1:]
|
|
if len(args) == 0 {
|
|
usage()
|
|
return fmt.Errorf("no command given")
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
// Whatever this process holds of the controller's lease is given back as it ends (novox/hq to-be
|
|
// 45 §6), so the next controller takes it at once rather than after its age.
|
|
defer theLease.release()
|
|
|
|
switch args[0] {
|
|
case "build":
|
|
return buildCommand(ctx, args[1:])
|
|
case "builder":
|
|
return builderCommand(ctx, args[1:])
|
|
case "board":
|
|
return boardCommand(ctx, args[1:])
|
|
case "api":
|
|
return apiCommand(ctx, args[1:])
|
|
case "licence":
|
|
return licenceCommand(ctx, args[1:])
|
|
case "rotate":
|
|
return rotateCommand(ctx, args[1:])
|
|
case "ask":
|
|
return askCommand(ctx, args[1:])
|
|
case "builds":
|
|
return buildsCommand(ctx, args[1:])
|
|
// The build queue, controlled by hand (novox/hq ADR 0219).
|
|
case "queue":
|
|
return queueCommand(ctx, args[1:])
|
|
case "cancel":
|
|
return cancelCommand(ctx, args[1:])
|
|
case "clear":
|
|
return clearCommand(ctx, args[1:])
|
|
case "rebuild":
|
|
return rebuildCommand(ctx, args[1:])
|
|
case "replay":
|
|
return replayCommand(ctx, args[1:])
|
|
case "kill":
|
|
return killCommand(ctx, args[1:])
|
|
case "pause", "resume":
|
|
return pauseCommand(ctx, args[0], args[1:])
|
|
case "collection":
|
|
return collectionCommand(ctx, args[1:])
|
|
case "plans":
|
|
return plansCommand(ctx, args[1:])
|
|
case "pin":
|
|
return pinCommand(ctx, args[1:], true)
|
|
case "unpin":
|
|
return pinCommand(ctx, args[1:], false)
|
|
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
|
|
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
|
|
// own schema is not a special case. `migrate` remains the word a person types.
|
|
case "prepare", "migrate":
|
|
return migrate(ctx)
|
|
case "node":
|
|
return nodeCommand(ctx, args[1:])
|
|
case "token":
|
|
return tokenCommand(ctx, args[1:])
|
|
case "identity":
|
|
return identityCommand(ctx, args[1:])
|
|
case "broker":
|
|
return brokerCommand(ctx, args[1:])
|
|
case "serve":
|
|
return serve(ctx)
|
|
case "upgrade":
|
|
return upgradeCommand(ctx, args[1:])
|
|
case "declare":
|
|
return declare(ctx, args[1:])
|
|
case "overlay":
|
|
return overlayCommand(ctx, args[1:])
|
|
case "module":
|
|
return moduleCommand(ctx, args[1:])
|
|
case "assign", "unassign":
|
|
return assignCommand(ctx, args[0], args[1:])
|
|
case "take":
|
|
return takeCommand(ctx, args[1:])
|
|
case "converge":
|
|
return convergeCommand(ctx, args[1:])
|
|
case "adopt":
|
|
return adoptCommand(ctx, args[1:])
|
|
case "settings":
|
|
return settingsCommand(ctx, args[1:])
|
|
case "secret":
|
|
return secretCommand(ctx, args[1:])
|
|
case "operator":
|
|
return operatorCommand(ctx, args[1:])
|
|
case "plan":
|
|
return planCommand(ctx, args[1:])
|
|
case "push":
|
|
return pushCommand(ctx, args[1:])
|
|
case "rollout":
|
|
return rolloutCommand(ctx, args[1:])
|
|
case "seats":
|
|
return seatsCommand(ctx, args[1:])
|
|
case "seat":
|
|
return seatCommand(ctx, args[1:])
|
|
case "status":
|
|
return statusCommand(ctx, args[1:])
|
|
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
|
case "hand-act":
|
|
return handActCommand(ctx, args[1:])
|
|
case "hand-acts":
|
|
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
|
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
|
case "durations":
|
|
return durationsCommand(ctx, args[1:])
|
|
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
|
case "conditions":
|
|
return conditionsCommand(ctx, args[1:])
|
|
case "doctor":
|
|
return doctorCommand(ctx, args[1:])
|
|
// What the healers did, and their brake (novox/hq to-be 45 §7).
|
|
case "healers":
|
|
return healersCommand(ctx, args[1:])
|
|
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
|
// (novox/hq ADR 0230).
|
|
case "retire":
|
|
return retireCommand(ctx, args[1:])
|
|
case "cleanup":
|
|
return cleanupCommand(ctx, args[1:])
|
|
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
|
case "data":
|
|
return dataCommand(ctx, args[1:])
|
|
case "version":
|
|
fmt.Println(version)
|
|
return nil
|
|
case "help", "-h", "--help":
|
|
usage()
|
|
return nil
|
|
default:
|
|
usage()
|
|
return fmt.Errorf("%q is not a command", args[0])
|
|
}
|
|
}
|
|
|
|
func usage() {
|
|
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
|
|
|
migrate bring each context's schema up to date
|
|
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
|
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
|
node list the nodes this mesh knows about
|
|
node show <name> what one machine reported it can do, and why
|
|
node public-domain <name> the domain it composes its routed names under
|
|
node public-domain <name> <d> ...set it to d
|
|
node public-domain <name> --clear ...it faces the outside no longer
|
|
node networks <name> the networks it routes for what it hosts
|
|
node networks <name> <cidr>... ...set them; its filter forwards these too
|
|
node networks <name> --clear ...only the container runtime's own
|
|
token issue --node <name> a one-time right to join, for an existing record
|
|
token issue --new <name> create the record and issue for it
|
|
token issue ... --adopted ...for a machine in use, which joins adopted
|
|
identity show this control plane's signing key
|
|
broker show where the broker is, and what to expect there
|
|
serve consume what nodes say, and answer
|
|
declare <node> <file> send a node a signed declaration
|
|
overlay place <node> [flags] say where a node is and how it is reached
|
|
overlay show the private network, as the mesh computes it
|
|
module add <file> register a module from its manifest
|
|
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
|
module list what modules this mesh knows about
|
|
module moved <name> <commit> the source has a newer commit than the mesh built
|
|
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
|
module forget <name> --and-what-it-holds ...and discard its settings, secrets and ports too
|
|
module issue <name> --node <m> a broker account for a module, scoped to its emits and consumes
|
|
upgrade <name> what happens when this module's current version moves
|
|
upgrade <name> roll-out [--together] ...send it to the machines running it
|
|
upgrade <name> record ...record that they are behind, and send nothing
|
|
status [--json] what is wrong, what is quiet, and what is out of date
|
|
retire [--json] every provider waiting for a person to approve a retirement (ADR 0230)
|
|
retire approve <node> <module> --why <text> retire what it waits with: access off, data kept
|
|
retire reject <node> <module> --why <text> keep them active; a warning stays open
|
|
cleanup [list] [--json] every retired consumer per provider: age, size, why
|
|
cleanup delete <node> <module> <consumer> --why <text> the provider deletes that one retired consumer
|
|
cleanup delete --older-than <days> --why <text> [--confirm] list those older; delete only with --confirm
|
|
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
|
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
|
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
|
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
|
board [--listen ADDR] the same three questions, as a page that holds nothing
|
|
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
|
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
|
unassign <node> <module>... take them off
|
|
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
|
what it declares; --yes <digest> cuts it over as previewed
|
|
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
|
adopt <node> return a converged node to adopted; what was taken stays taken
|
|
settings set <module> <file> what a module's config should say, for the whole mesh
|
|
settings set <module> <file> --node <n> ...or for one machine
|
|
settings clear <module> [--node <n>] take a layer away
|
|
secret accept <node> <module> <name> carry a value the mesh did not make and cannot invent
|
|
secret accept ... --from <file> ...read it from a file rather than being asked
|
|
secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
|
|
break-glass: open the operator-sealed copy, to a 0600 file
|
|
secret export [--out <file>] every operator-sealed copy, ciphertext — keep it with the key
|
|
operator key make [--out <file>] make the operator's sealing key, off the mesh; private half to the file only
|
|
operator key set <public> [--replace] tell the mesh which operator key to seal to
|
|
operator key show the operator key, and what it can recover
|
|
build <repository> [--ref R] have a build machine build it, and record what came out
|
|
build --behind build every module the mesh holds older than its source
|
|
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
|
builds [<module>] what has been built lately, and what came of it
|
|
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
|
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
|
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
|
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
|
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
|
kill <id> end a build where it runs; recorded failed, killed by hand
|
|
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
|
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
|
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
|
record an act done by hand outside the mesh (to-be 45 §7)
|
|
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
|
conditions [--scope S] [--severity S] [--machine M] [--json]
|
|
what is wrong now: every open condition, urgent first (to-be 45 §2)
|
|
conditions show <key> one condition whole, with its evidence
|
|
conditions silence <key> --for <d> --why <text> send no message for it a while; a hand act
|
|
conditions history [--days N] [--key K] every raising, change and clearing lately
|
|
doctor [run|probes|signals] [--json]
|
|
the self-check: the last verdict, a run now, the probes, the signals' ages
|
|
healers [--days N] [--json] the healers, what they did lately, and their brake (to-be 45 §7)
|
|
durations [--kind K] [--days N] [--json]
|
|
apply, heartbeat, plan-tier and build durations, per machine or module
|
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
|
delivered as the builder module's broker secret (module add it first)
|
|
licence add|list|use|key model access, under the name a person calls it
|
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
|
licence refresh <name> mint a new access token and seal it to every holder
|
|
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
|
pin <node> <provision> <from-node> <module>
|
|
which provider this one gets a provision from: the module, and its node
|
|
unpin <node> <provision> put that question back
|
|
plan <node> [--files|--json] what that node would run, and why
|
|
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
|
that need it; --why records it in the hand-act log
|
|
version what this binary is
|
|
|
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
|
`+store.Variable("<context>")+` — or `+store.FileVariable("<context>")+`, naming a file that holds
|
|
the same thing and keeps the password out of the environment. This process holds:
|
|
|
|
`)
|
|
for _, c := range held {
|
|
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
|
c.name, store.Database(c.name), store.Variable(c.name))
|
|
}
|
|
fmt.Fprintln(os.Stderr)
|
|
}
|
|
|
|
// parseAround reads flags that may sit before, after or between positional arguments.
|
|
//
|
|
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
|
|
// parses no flags at all and silently ignores every one of them. The host learned this the same
|
|
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
|
|
// keeps naming, and it looks exactly like success.
|
|
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
|
var positionals []string
|
|
rest := args
|
|
for {
|
|
if err := set.Parse(rest); err != nil {
|
|
return nil, err
|
|
}
|
|
rest = set.Args()
|
|
if len(rest) == 0 {
|
|
return positionals, nil
|
|
}
|
|
positionals = append(positionals, rest[0])
|
|
rest = rest[1:]
|
|
}
|
|
}
|
|
|
|
// Built is the daemon hearing a build's outcome on the bus — its own asking, an announcement's, or
|
|
// a tool's that did not wait (novox/hq issue 176) — and taking it in: recorded, and the module
|
|
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
|
// became of a build nobody was watching.
|
|
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
|
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
|
// before anyone had reviewed it.
|
|
if result.DryRun {
|
|
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
|
return nil
|
|
}
|
|
manifest, _, err := takeIn(ctx, b.inv, result)
|
|
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
|
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
|
asked, _ := link.BuildAskedAt(result.ID)
|
|
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
|
// Phase 0). Said if lost; never a reason not to take the build in.
|
|
recordBuildDuration(ctx, b.inv, result, asked)
|
|
switch {
|
|
case err != nil && result.Failed != "":
|
|
fmt.Printf("%s: %v\n", result.ID, err)
|
|
if result.Module != "" {
|
|
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
|
} else {
|
|
planFailedBuild(ctx, b.open, result)
|
|
}
|
|
return nil
|
|
case errors.Is(err, inventory.ErrSuperseded):
|
|
// Not a failure: the module is already at what a later request built. A plan that asked
|
|
// before that later request is answered by it; one that asked after it ignores this.
|
|
fmt.Printf("%s: %v\n", result.ID, err)
|
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
|
return nil
|
|
case err != nil:
|
|
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
|
if manifest.Module != "" {
|
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
|
}
|
|
return nil
|
|
}
|
|
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
|
statusFrom.nudge()
|
|
return nil
|
|
}
|