novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Rename mesh-control -> mesh-controller, substrate -> foundation

Browse Source 
One name per thing, per the HQ glossary: the module/container/image/binary/repo
becomes mesh-controller, the seat the-controller, and the store+broker pair the
foundation (embedded base bundles, default template and example lock renamed with
their go:embed directives). No behaviour change — a pure vocabulary rename.

Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx
This commit is contained in:
jschoubben
2026-09-16 18:40:40 +02:00
parent 366840fc0d
commit c3b88b9148
77 changed files with 157 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/acts.go
+125
View File
@@ -0,0 +1,125 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The things the mesh can be asked to do, separated from how it was asked.
//
// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and
// the command API call the same functions here, so an assignment refused at one is refused at the
// other for the same reason and in the same words. The moment a surface can accept something
// another would reject, the mesh has two answers to one question and people learn which to trust.
//
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
// composes its own explanation, because two explanations of one refusal drift.
// assign puts a module on a node, and says at once what in the mesh no longer works out.
//
// The assignment is kept even when the node does not resolve: it is what a person meant, and
// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
// long as it takes to assign the provider, and refusing the first half of a pair would make the
// order somebody types two commands in part of the mesh's rules.
//
// **What is checked is the mesh, not the one machine** — because that is what the assignment
// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
// action tested one node and the verify is resolution over all of them, so an assignment could be
// reported as fine while it took a provision away from every other machine — a module offering a
// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
// module already assigned. That was found on a four-node raise and read as a version bump breaking
// provider recognition; it was neither the version nor the provider.
//
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
// machines this just blocked is worth more than the milliseconds.
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
if err := open.inventory.Assign(ctx, node, module); err != nil {
return "", err
}
said := fmt.Sprintf("%s is assigned %s", node, module)
plan, _, err := planFor(ctx, open, node)
if err != nil {
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
// worth reporting: this machine's refusal is rarely the only consequence.
return said + blockedElsewhere(ctx, open, node), err
}
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
// capability the machine lacks is on the wrong machine, and the assignment records what a person
// meant while this line says it will not run until it moves. The rest of the node still pushes.
for _, u := range plan.Unhostable {
if u.Module != module {
continue
}
for _, c := range u.Missing {
said += "\n but " + catalogue.WrongMachine(u.Module, c, node)
}
}
return said + fmt.Sprintf("\n run `push %s` to send it", node) +
blockedElsewhere(ctx, open, node), nil
}
// unassign takes a module off a node. What it leaves behind is the host's business: a directory
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
//
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
// module off one machine is the ordinary way to stop providing something to another, and nothing
// about the command's own output would ever have said so.
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
if err := open.inventory.Unassign(ctx, node, module); err != nil {
return "", err
}
return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
node, module, node) + blockedElsewhere(ctx, open, node), nil
}
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
//
// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
// whole mesh twice and would still be a guess about which of several changes did it; saying
// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
// and cannot mislead. The machine that was just changed is left out because its own refusal is
// already the answer beside this one.
//
// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
// not a reason to claim the assignment did not happen — it did.
func blockedElsewhere(ctx context.Context, open *stores, except string) string {
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return "\n\nThe rest of the mesh could not be checked: " + err.Error()
}
blocked := map[string]string{}
for _, n := range nodes {
if n.Name == except {
continue
}
if _, _, err := planFor(ctx, open, n.Name); err != nil {
blocked[n.Name] = err.Error()
}
}
if len(blocked) == 0 {
return ""
}
names := make([]string, 0, len(blocked))
for name := range blocked {
names = append(names, name)
}
sort.Strings(names)
var out strings.Builder
fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
"nothing will be sent to them:\n", len(names))
for _, name := range names {
fmt.Fprintf(&out, " %s\n", name)
for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
}
}
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
return out.String()
}
cmd/mesh-controller/acts_test.go
+133
View File
@@ -0,0 +1,133 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// What an assignment says about the machines it was not about.
// **An assignment that blocks another machine says so.**
//
// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it
// the moment its own node stops resolving, so an assignment to the provider's machine silently took
// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this
// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the
// way back, as a version bump breaking provider recognition. It was neither the version nor the
// provider: it was one machine's set of assignments, and nothing said so.
//
// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify
// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the
// verify is resolution over all of them.
func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
// A provider on the hub and a consumer on the other machine, both resolving.
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
Requires: []string{"acme-ca"}})
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
t.Fatal(err)
}
said, err := assign(ctx, open, "laptop", "route-proxy")
if err != nil {
t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said)
}
if strings.Contains(said, "cannot be worked out") {
t.Fatalf("a well mesh was reported as blocked:\n%s", said)
}
// Now break the hub's own set, with nothing but the command a person has.
one, two := rivals()
register(t, open, one)
register(t, open, two)
if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil {
t.Fatal(err)
}
said, err = assign(ctx, open, "anchor", "rival-two")
if err == nil {
t.Fatal("an assignment that makes its own node incoherent was not reported at all")
}
// The node's own refusal is the error, as it always was. What is new is that the machines this
// just took a provision away from are named in the same breath.
if !strings.Contains(said, "laptop") {
t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err)
}
if !strings.Contains(said, "acme-ca") {
t.Fatalf("what laptop is now missing is not said:\n%s", said)
}
if !strings.Contains(said, "cannot be worked out as things stand") {
t.Fatalf("the report does not say what state the mesh is in:\n%s", said)
}
// And the assignment is kept: it is what a person meant, and assignment is not an ordering.
assigned, err := open.inventory.Assigned(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if !contains(assigned, "rival-two") {
t.Fatalf("the assignment was not kept: %v", assigned)
}
}
// A consumer assigned before its provider is refused for itself and kept, because assignment is not
// an ordering — refusing the first half of a pair would make the order somebody types two commands
// in part of the mesh's rules.
func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
Requires: []string{"acme-ca"}})
said, err := assign(ctx, open, "laptop", "route-proxy")
if err == nil {
t.Fatalf("a consumer with nothing to consume resolved:\n%s", said)
}
assigned, err := open.inventory.Assigned(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if !contains(assigned, "route-proxy") {
t.Fatalf("assignment became an ordering: %v", assigned)
}
}
// Unassigning is the ordinary way to stop providing something to another machine, and it reports
// the same way for the same reason.
func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
Requires: []string{"acme-ca"}})
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
t.Fatal(err)
}
said, err := unassign(ctx, open, "anchor", "step-ca")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") {
t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said)
}
}
func contains(all []string, one string) bool {
for _, s := range all {
if s == one {
return true
}
}
return false
}
cmd/mesh-controller/api.go
+166
View File
@@ -0,0 +1,166 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"net/http"
"strings"
"time"
)
// The command API: what the mesh can be asked to do, over a network.
//
// **An adapter and nothing else** (novox/hq ADR 0035). Every route here calls the same function
// the command line calls, so a refusal is the same refusal in the same words. Nothing is decided
// in this file — the moment it validates something the command line does not, the mesh has two
// answers to one question.
//
// **It refuses everything unless it was told how to know who is asking.** The board is published
// on a public name, and this is what stands behind it: an unauthenticated command surface reachable
// from the internet is authority over the mesh handed to whoever finds it. So there is no
// permissive default and no flag that removes the check — a mesh that has not been told how to
// authenticate serves nothing, loudly.
//
// The intended authenticator is an OAuth2 provider (ADR 0035), which is an ordinary module. Until
// one is configured this refuses, which is the correct behaviour rather than a placeholder: a
// surface that worked without authentication would be one somebody left running.
func apiCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("api", flag.ContinueOnError)
listen := set.String("listen", "127.0.0.1:8081", "where to serve it")
issuer := set.String("issuer", "",
"the OAuth2 issuer whose tokens this accepts; without it, nothing is served")
if _, err := parseAround(set, args); err != nil {
return err
}
if strings.TrimSpace(*issuer) == "" {
// Refused at start rather than per request, so it is discovered by whoever ran it rather
// than by whoever finds it.
return errors.New(
"--issuer is not set, and this serves commands rather than pages: it will not run " +
"without being told whose tokens to believe. An OAuth2 provider is an ordinary " +
"module (novox/hq ADR 0035)")
}
server := &http.Server{
Addr: *listen,
ReadHeaderTimeout: 10 * time.Second,
Handler: commands(mustAuthenticate(*issuer)),
}
fmt.Printf("the command API is on http://%s\n", *listen)
fmt.Printf(" it accepts tokens from %s and refuses everything else\n", *issuer)
fmt.Printf(" every route calls what the command line calls\n")
go func() {
<-ctx.Done()
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = server.Shutdown(closing)
}()
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
// Authenticator says whether a request may act, and as whom.
//
// An interface so the check can be driven by a test without an identity provider, and so the one
// real implementation is the only thing that has to be right.
type Authenticator interface {
// Who returns the subject a request is acting as, or an error naming why it may not.
Who(r *http.Request) (string, error)
}
// mustAuthenticate is the real one: a bearer token from the configured issuer.
//
// **Not yet verifying the signature**, and it says so rather than pretending. Verification needs
// the issuer's keys, which needs an identity provider to exist — so this refuses every request
// until that is built, which is the same answer as having no API at all and is honest about why.
func mustAuthenticate(issuer string) Authenticator { return notYet{issuer: issuer} }
type notYet struct{ issuer string }
func (n notYet) Who(*http.Request) (string, error) {
return "", fmt.Errorf(
"this mesh has no way to verify a token from %s yet: the identity provider is a module "+
"and none is running. Use the command line, which authenticates through nothing "+
"because it is already behind the machine's own login", n.issuer)
}
// commands is the routing, separate so a test can drive it without a listener.
func commands(who Authenticator) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("POST /assign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
return assign(ctx, open, in.Node, in.Module)
}))
mux.HandleFunc("POST /unassign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
return unassign(ctx, open, in.Node, in.Module)
}))
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
// expected is indistinguishable from one that is down.
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
refuse(w, http.StatusNotFound, fmt.Errorf(
"%s %s is not something this mesh can be asked; it accepts POST /assign and "+
"POST /unassign", r.Method, r.URL.Path))
})
return mux
}
type request struct {
Node string `json:"node"`
Module string `json:"module"`
}
// acting is the shape every route shares: authenticate, read, act, answer.
func acting(
who Authenticator,
do func(context.Context, *stores, request) (string, error),
) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if _, err := who.Who(r); err != nil {
refuse(w, http.StatusUnauthorized, err)
return
}
var in request
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
return
}
if in.Node == "" || in.Module == "" {
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
return
}
open, err := openStores(r.Context())
if err != nil {
refuse(w, http.StatusServiceUnavailable, err)
return
}
defer open.Close()
said, err := do(r.Context(), open, in)
if err != nil {
// **The refusal the command line would have given, unchanged.** Carrying `said` with
// it matters: an assignment that was kept and still does not resolve is two facts,
// and dropping either makes the answer wrong.
answer(w, http.StatusConflict, map[string]any{"said": said, "refused": err.Error()})
return
}
answer(w, http.StatusOK, map[string]any{"said": said})
}
}
func answer(w http.ResponseWriter, status int, body map[string]any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
func refuse(w http.ResponseWriter, status int, err error) {
answer(w, status, map[string]any{"refused": err.Error()})
}
cmd/mesh-controller/api_test.go
+77
View File
@@ -0,0 +1,77 @@
package main
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
type letIn struct{}
func (letIn) Who(*http.Request) (string, error) { return "somebody", nil }
func asking(t *testing.T, who Authenticator, method, path, body string) *httptest.ResponseRecorder {
t.Helper()
recorded := httptest.NewRecorder()
commands(who).ServeHTTP(recorded, httptest.NewRequest(method, path, strings.NewReader(body)))
return recorded
}
// **Nothing is served to somebody the mesh cannot identify**, and that is the default rather than
// a setting. The board this stands behind is published on a public name (novox/hq 11-a-board), so
// an unauthenticated command surface is authority over the mesh handed to whoever finds it.
func TestAnUnauthenticatedRequestIsRefused(t *testing.T) {
got := asking(t, mustAuthenticate("https://identity.example/realms/mesh"),
"POST", "/assign", `{"node":"anchor","module":"umami"}`)
if got.Code != http.StatusUnauthorized {
t.Fatalf("an unidentified caller got %d", got.Code)
}
// And it says what to do instead, because the answer is not "give up".
if !strings.Contains(got.Body.String(), "command line") {
t.Errorf("the refusal does not say what still works: %s", got.Body.String())
}
}
// The API refuses to start at all without being told whose tokens to believe.
//
// At start rather than per request, so it is found by whoever ran it rather than by whoever
// finds it.
func TestTheApiWillNotRunWithoutAnIssuer(t *testing.T) {
err := apiCommand(context.Background(), []string{})
if err == nil {
t.Fatal("it served commands without being told who may give them")
}
if !strings.Contains(err.Error(), "issuer") {
t.Errorf("the refusal does not name what is missing: %v", err)
}
}
// A request missing what it acts on is refused before anything is opened.
func TestARequestThatNamesNothingIsRefused(t *testing.T) {
// **The exact refusal, not merely "not accepted".** Asserting non-200 passes even when the
// request got as far as opening a store and failing there, which proves nothing about whether
// anything was checked — that is what the first version of this test did.
for _, body := range []string{`{}`, `{"node":"anchor"}`, `{"module":"umami"}`, `not json`} {
got := asking(t, letIn{}, "POST", "/assign", body)
if got.Code != http.StatusBadRequest {
t.Errorf("%s got %d, and a request naming nothing is a bad request", body, got.Code)
}
}
}
// A verb nobody implemented is said plainly. A command surface answering 404 to something
// somebody expected is indistinguishable from one that is down.
func TestAnUnknownRouteSaysWhatIsAccepted(t *testing.T) {
got := asking(t, letIn{}, "POST", "/rotate", `{}`)
if got.Code != http.StatusNotFound {
t.Fatalf("got %d", got.Code)
}
var said map[string]any
_ = json.Unmarshal(got.Body.Bytes(), &said)
if !strings.Contains(said["refused"].(string), "/assign") {
t.Errorf("it does not say what it does accept: %v", said)
}
}
cmd/mesh-controller/board.go
+317
View File
@@ -0,0 +1,317 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"html/template"
"net/http"
"sort"
"strings"
"time"
)
// boardCommand serves the three questions as a page.
//
// **It reads through the same functions everything else does and holds nothing**
// (novox/hq 03-DESIGN/01-to-be/11-a-board.md). The board being replaced is one service that reads
// every context's database directly — [ADR 0008](novox/hq) violated by the one component with a
// reason to violate it, and the cost is that a boundary nothing may cross can move, while one
// thing crossing it is enough to freeze it. A board that reads the provisioning tables is a board
// that breaks when provisioning changes its tables, and the change then gets weighed against the
// board.
//
// **It stores nothing of its own.** No cache that can disagree, no table of what the mesh looked
// like last time. Every request reads the mesh now; if that is slow, the answer belongs in the
// context that owns it, where everything else asking gets it too.
//
// **Reading is the whole of it.** Every action a board could offer already exists as a command,
// and a button that does something no command does is a second implementation of a decision.
func boardCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("board", flag.ContinueOnError)
// The private network, not everything. A board says which machines are broken and what they
// are running, which is exactly the map somebody attacking this would like — and there is no
// reason for it to be reachable from further away than the mesh.
listen := set.String("listen", "127.0.0.1:8080", "where to serve it")
if _, err := parseAround(set, args); err != nil {
return err
}
server := &http.Server{
Addr: *listen,
ReadHeaderTimeout: 10 * time.Second,
Handler: board(),
}
fmt.Printf("the board is on http://%s\n", *listen)
fmt.Printf(" it reads the mesh on every request and keeps nothing\n")
go func() {
<-ctx.Done()
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = server.Shutdown(closing)
}()
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
// board is the handler, separate so a test can drive it without a listener.
func board() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
asked, err := ask(r.Context())
if err != nil {
// **Said, not blank.** A board that cannot reach the mesh and renders an empty page
// says "nothing is wrong" in the one situation where nobody can know that.
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusServiceUnavailable)
_ = page.Execute(w, view{Unreachable: err.Error()})
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := page.Execute(w, asked); err != nil {
// The page is half-written by now; there is nothing useful left to say to the
// browser, and saying it here is what stops the failure being silent.
fmt.Printf("the board could not render: %v\n", err)
}
})
// The same answers for something that is not a person, from the same read. A board and a
// script disagreeing about which machine is broken would be worse than either alone.
mux.HandleFunc("/mesh.json", func(w http.ResponseWriter, r *http.Request) {
open, err := openStores(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
defer open.Close()
asked, err := theThreeQuestions(r.Context(), open)
if err != nil {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
body, err := statusAsJSON(asked)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(append(body, '\n'))
})
return mux
}
// ask reads the mesh for one request.
func ask(ctx context.Context) (view, error) {
open, err := openStores(ctx)
if err != nil {
return view{}, err
}
defer open.Close()
asked, err := theThreeQuestions(ctx, open)
if err != nil {
return view{}, err
}
return viewOf(asked), nil
}
// view is what the page is given. Nothing is derived here that the reader could not derive.
type view struct {
Unreachable string
Machines int
Broken []brokenMachine
Quiet []quietMachine
Behind []staleModule
Waiting []waitingMachine
// Unresolved is every machine that cannot be worked out at all. Shown above everything else,
// because a machine here is in none of the other lists: nothing was computed for it, so it is
// not broken, not quiet and not behind — and a page without this said "all well" about a mesh
// where nothing could be sent anywhere.
Unresolved []blockedMachine
// Network is why the private network could not be computed, when it could not.
Network string
At string
}
type blockedMachine struct {
Node string
// Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
// a machine is usually blocked by more than one and fixing one of them changes nothing.
Said []string
}
type waitingMachine struct {
Node string
// Never told is not out of date: nobody has ever asked this machine to be anything. Same
// remedy, different situation, and the page says which.
Never bool
}
type brokenMachine struct {
Node string
// Outcome is refused or failed, and stays distinct all the way to the page. **Refused means
// the machine is exactly as it was and what is wrong is in what was sent; failed means it is
// in a state nobody declared and what is wrong is on the machine.** They are fixed in
// different places, so one word for both would send half the readers to the wrong one.
Outcome string
Said []string
When string
}
type quietMachine struct {
Node string
// Heard is "never" or how long ago. Never heard from is not the same as quiet for a while:
// one may be a machine that was never sent anything.
Heard string
}
type staleModule struct {
Module string
Holds string
Source string
Running []string
}
func viewOf(asked answers) view {
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
Network: asked.network}
var blocked []string
for name := range asked.refused {
blocked = append(blocked, name)
}
sort.Strings(blocked)
for _, name := range blocked {
one := blockedMachine{Node: name}
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
one.Said = append(one.Said, strings.TrimSpace(line))
}
out.Unresolved = append(out.Unresolved, one)
}
for _, d := range asked.wrong {
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
When: d.At.Local().Format("2006-01-02 15:04")}
if d.Refused != "" {
// The host's own words. It says exactly what it could not accept, and nothing
// written here would say it better.
one.Said = append(one.Said, firstLine(d.Refused))
}
for _, f := range d.Failed {
one.Said = append(one.Said, f.ID+": "+firstLine(f.Error))
}
out.Broken = append(out.Broken, one)
}
for _, n := range asked.quiet {
out.Quiet = append(out.Quiet, quietMachine{Node: n.Name, Heard: heardFrom(n)})
}
for _, m := range asked.waiting {
out.Waiting = append(out.Waiting, waitingMachine{Node: m.Node, Never: m.Never})
}
for module, on := range asked.behind {
from := asked.sources[module]
out.Behind = append(out.Behind, staleModule{
Module: module, Holds: short(from.BuiltFrom), Source: short(from.Head), Running: on,
})
}
return out
}
// The page. Deliberately one file with no assets: a board that cannot render without fetching
// something is a board that is blank exactly when the mesh is unwell.
var page = template.Must(template.New("board").Parse(`<!doctype html>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>the mesh</title>
<style>
:root { color-scheme: light dark; }
body { font: 15px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; margin: 2rem auto;
max-width: 52rem; padding: 0 1rem; }
h1 { font-size: 1.1rem; font-weight: 600; margin: 0 0 1.5rem; }
h2 { font-size: 1rem; font-weight: 600; margin: 2rem 0 .5rem; }
.quiet { opacity: .65; }
.said { opacity: .8; padding-left: 1.5rem; }
.outcome { display: inline-block; min-width: 4.5rem; }
.refused { color: #b26b00; }
.failed { color: #c0392b; }
ul { list-style: none; padding: 0; margin: 0; }
li { padding: .15rem 0; }
footer { margin-top: 3rem; opacity: .6; font-size: .85rem; }
</style>
{{if .Unreachable}}
<h1>the mesh cannot be read</h1>
<p class="failed">{{.Unreachable}}</p>
<p class="quiet">This says nothing about whether the mesh is well — only that this page could not
find out.</p>
{{else}}
<h1>{{.Machines}} machine{{if ne .Machines 1}}s{{end}}</h1>
{{if .Unresolved}}
<h2>Can everything be worked out?</h2>
<ul>
{{range .Unresolved}}
<li><span class="outcome failed">blocked</span> <strong>{{.Node}}</strong>
{{range .Said}}<div class="said">{{.}}</div>{{end}}
</li>
{{end}}
</ul>
<p class="quiet">Nothing can be sent to a machine here, and it appears in none of the lists below:
nothing was computed for it, so there is nothing it can be behind.</p>
{{end}}
{{if .Network}}
<p class="failed">The private network could not be computed: {{.Network}}</p>
{{end}}
<h2>Is anything broken?</h2>
{{if .Broken}}
<ul>
{{range .Broken}}
<li>
<span class="outcome {{.Outcome}}">{{.Outcome}}</span>
<strong>{{.Node}}</strong> <span class="quiet">{{.When}}</span>
{{range .Said}}<div class="said">{{.}}</div>{{end}}
</li>
{{end}}
</ul>
{{else}}<p class="quiet">No. Every machine is doing what it was told.</p>{{end}}
<h2>Is anything not answering?</h2>
{{if .Quiet}}
<ul>{{range .Quiet}}<li><strong>{{.Node}}</strong> <span class="quiet">{{.Heard}}</span></li>{{end}}</ul>
<p class="quiet">Not heard from is not the same as tried and could not — a machine here may be
new, switched off, or unreachable.</p>
{{else}}<p class="quiet">No. Every machine has been heard from.</p>{{end}}
<h2>Is anything out of date?</h2>
{{if .Behind}}
<ul>
{{range .Behind}}
<li><strong>{{.Module}}</strong> holds {{.Holds}}, source has {{.Source}}
{{if .Running}}<div class="said">running on {{range $i, $n := .Running}}{{if $i}}, {{end}}{{$n}}{{end}}</div>
{{else}}<div class="said quiet">assigned to nothing</div>{{end}}
</li>
{{end}}
</ul>
{{else}}<p class="quiet">No. Every module is what its source last had.</p>{{end}}
{{if .Waiting}}
<ul>
{{range .Waiting}}
<li><strong>{{.Node}}</strong>
{{if .Never}}<span class="quiet">has never been sent anything</span>
{{else}}<span class="quiet">is not running what the mesh would send it</span>{{end}}
</li>
{{end}}
</ul>
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
Both are sent by <code>push --behind</code>.</p>
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
{{end}}
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
`))
cmd/mesh-controller/board_test.go
+154
View File
@@ -0,0 +1,154 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// Refused and failed stay distinct all the way to the page.
//
// **Refused means the machine is exactly as it was and what is wrong is in what was sent; failed
// means it is in a state nobody declared and what is wrong is on the machine.** They are fixed in
// different places, so a page saying "error" for both sends half its readers to the wrong one.
func TestRefusedAndFailedReachThePageAsDifferentThings(t *testing.T) {
rendered := render(t, viewOf(answers{
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
wrong: []inventory.Doing{
{Node: "anchor", Outcome: "refused", Refused: "not a declaration this host speaks",
At: time.Now()},
{Node: "laptop", Outcome: "failed", At: time.Now(),
Failed: []inventory.FailedResource{{ID: "web.container", Error: "no such image"}}},
},
}))
// The rendered outcome, not the word anywhere on the page: both words appear in the
// stylesheet, so a plain Contains passes whatever the machine actually said. It did.
for _, want := range []string{`<span class="outcome refused">refused</span>`,
`<span class="outcome failed">failed</span>`} {
if !strings.Contains(rendered, want) {
t.Fatalf("the page does not distinguish the two — missing %s:\n%s", want, rendered)
}
}
// And the host's own words, which say exactly what it could not accept.
for _, want := range []string{"not a declaration this host speaks", "web.container", "no such image"} {
if !strings.Contains(rendered, want) {
t.Fatalf("the page does not say %q, so a reader must go and ask:\n%s", want, rendered)
}
}
}
// Nothing wrong is said, not left blank. An empty page and a well mesh must not look alike.
func TestAWellMeshSaysSoRatherThanShowingNothing(t *testing.T) {
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
for _, want := range []string{
"Every machine is doing what it was told",
"Every machine has been heard from",
"Every module is what its source last had",
} {
if !strings.Contains(rendered, want) {
t.Fatalf("a well mesh does not say %q:\n%s", want, rendered)
}
}
}
// A board that cannot reach the mesh must not render an empty page: that says "nothing is wrong"
// in the one situation where nobody can know it.
func TestABoardThatCannotReadTheMeshSaysSo(t *testing.T) {
rendered := render(t, view{Unreachable: "the store did not answer"})
if !strings.Contains(rendered, "the store did not answer") {
t.Fatalf("the reason is not on the page:\n%s", rendered)
}
if strings.Contains(rendered, "Every machine is doing what it was told") {
t.Fatal("a board that could not read the mesh reported that the mesh is well")
}
}
// Quiet is not broken, and the page says which it is.
func TestQuietIsNotReportedAsBroken(t *testing.T) {
rendered := render(t, viewOf(answers{
nodes: []inventory.Node{{Name: "laptop"}},
quiet: []inventory.Node{{Name: "laptop"}},
}))
if !strings.Contains(rendered, "not the same as tried and could not") {
t.Fatalf("the page does not separate quiet from broken:\n%s", rendered)
}
if !strings.Contains(rendered, "Every machine is doing what it was told") {
t.Fatalf("a quiet machine was counted as broken:\n%s", rendered)
}
}
// The page renders what a machine said, and a hostile string in it is not markup.
//
// What is on this page comes from machines, and a machine's own words are the whole reason the
// page is useful. They are also the one thing here that nobody in this repository wrote.
func TestWhatAMachineSaidIsNotMarkup(t *testing.T) {
rendered := render(t, viewOf(answers{
nodes: []inventory.Node{{Name: "anchor"}},
wrong: []inventory.Doing{{Node: "anchor", Outcome: "refused",
Refused: `<script>alert("from the machine")</script>`, At: time.Now()}},
}))
if strings.Contains(rendered, "<script>alert") {
t.Fatalf("a machine's words were rendered as markup:\n%s", rendered)
}
if !strings.Contains(rendered, "&lt;script&gt;") {
t.Fatalf("the words were not shown at all, so the reader cannot see what it said:\n%s", rendered)
}
}
// A module behind its source names the machines running the old one — the part with consequences.
func TestBeingBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
rendered := render(t, viewOf(answers{
nodes: []inventory.Node{{Name: "laptop"}},
behind: map[string][]string{"web": {"laptop"}},
sources: map[string]inventory.Source{"web": {BuiltFrom: "aaaaaaaaaa", Head: "bbbbbbbbbb"}},
}))
for _, want := range []string{"web", "aaaaaaaa", "bbbbbbbb", "running on laptop"} {
if !strings.Contains(rendered, want) {
t.Fatalf("the page does not say %q:\n%s", want, rendered)
}
}
}
func render(t *testing.T, v view) string {
t.Helper()
var out strings.Builder
if err := page.Execute(&out, v); err != nil {
t.Fatal(err)
}
return out.String()
}
// The board and the command must answer the same question the same way.
//
// A machine not running what the mesh would send it was added to `status` and would have been
// missing here — which is the one thing this page's design forbids: two answers to the same
// question, disagreeing, with a person in front of each.
func TestTheBoardSaysWhichMachinesHaveNotBeenSentWhatTheyShouldBe(t *testing.T) {
rendered := render(t, viewOf(answers{
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
waiting: []inventory.Machine{
{Node: "anchor"},
{Node: "laptop", Never: true},
},
}))
if !strings.Contains(rendered, "is not running what the mesh would send it") {
t.Fatalf("the page does not say a machine is out of date:\n%s", rendered)
}
// Never told is a different situation with the same remedy, and the page keeps them apart.
if !strings.Contains(rendered, "has never been sent anything") {
t.Fatalf("the page does not separate never told from out of date:\n%s", rendered)
}
if !strings.Contains(rendered, "push --behind") {
t.Fatalf("the page does not say what sends them:\n%s", rendered)
}
}
// And says so when there is nothing waiting, rather than leaving the question unanswered.
func TestAMeshWithNothingWaitingSaysSo(t *testing.T) {
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
if !strings.Contains(rendered, "Every machine is running what the mesh would send it") {
t.Fatalf("the page leaves the question unanswered:\n%s", rendered)
}
}
cmd/mesh-controller/build.go
+493
View File
@@ -0,0 +1,493 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// asking a build machine for a module, and what came back.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// buildCommand builds a module from its source and records what came out.
//
// **Run where there is a container runtime**, which is why it is a command rather than something
// the control plane does on its own: building needs to run things on a machine, and what the
// control plane may send a machine is bounded by the declaration language. This is the shape the
// builder module will take when it is given work over the broker; today a person runs it, and the
// mesh records the result the same way either way.
func buildCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("build", flag.ContinueOnError)
ref := set.String("ref", "", "the branch, tag or commit to build")
// A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's
// root, which is the ordinary case and why this is a flag rather than a second argument.
path := set.String("path", "", "the module's directory inside the repository")
wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
// Every module whose source has moved, rather than one named repository.
//
// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
// already knows which modules are behind their source, so making a person read that list and
// retype each repository is asking them to be the loop. Naming a repository and asking which
// ones need building are different requests, so they are not combined.
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if *behind {
if len(positionals) != 0 {
return errors.New("build <repository> or build --behind, not both: one names a " +
"repository and the other asks which need building")
}
return buildBehind(ctx, *wait)
}
if len(positionals) != 1 {
return errors.New("build <repository> [--ref R] [--wait D] [--dry-run]")
}
if *dryRun {
return buildAndShow(ctx, positionals[0], *path, *ref, *wait)
}
return buildOne(ctx, positionals[0], *path, *ref, *wait)
}
// buildFrom turns what a builder said into what the mesh keeps.
func buildFrom(result link.BuildResult) inventory.Build {
kept := inventory.Build{
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
Commit: result.Commit, On: result.On, Failed: result.Failed,
// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
// The catalogue turns the manifest into requires/provides edges and `against` into build
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path, Manifest: result.Manifest, Against: result.Against,
}
for _, made := range result.Made {
kept.Made = append(kept.Made, inventory.Artifact{
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
})
}
// The module name comes from the manifest, which only exists when the build got that far.
if len(result.Manifest) > 0 {
if m, err := catalogue.ParseManifest(result.Manifest); err == nil {
kept.Module = m.Module
}
}
return kept
}
// buildsCommand says what has been built lately.
func buildsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("builds", flag.ContinueOnError)
limit := set.Int("n", 20, "how many to show")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
module := ""
if len(positionals) == 1 {
module = positionals[0]
} else if len(positionals) > 1 {
return errors.New("builds [<module>] [-n N]")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
builds, err := inv.Builds(ctx, module, *limit)
if err != nil {
return err
}
if len(builds) == 0 {
// Said rather than printed as nothing: an empty list and a failed read must never look
// the same, and getting here means the store answered.
if module != "" {
fmt.Printf("nothing has been built for %s\n", module)
return nil
}
fmt.Println("nothing has been built yet")
return nil
}
for _, b := range builds {
what := b.Module
if what == "" {
// It failed before knowing what it was building, which is most of the interesting
// failures. The repository is what a person has to go and look at.
what = "?"
}
outcome := "built " + short(b.Commit)
if !b.Worked() {
outcome = "failed"
}
fmt.Printf("%-18s %-14s %-10s %s\n",
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"))
fmt.Printf(" %s", b.Repository)
if b.Ref != "" {
fmt.Printf(" at %s", b.Ref)
}
fmt.Println()
for _, made := range b.Made {
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
}
if !b.Worked() {
// The builder's own first line. The whole failure is often a build log, and printing
// it here would bury every other row.
fmt.Printf(" %s\n", firstLine(b.Failed))
}
}
return nil
}
// builderCommand issues a build machine its own broker credential.
//
// **A build machine is not a node**, and giving it a node's account would let it read another
// machine's declarations. This is narrower and different: read the build queue, write the
// exchange and an asker's reply queue, and nothing else.
//
// Issued rather than assumed, because until this the builder used whatever credential it was
// handed — which in practice meant the broker's own administrative one. A program documented as
// holding its own credential and given somebody else's is worse than one with no story at all.
func builderCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("builder issue", flag.ContinueOnError)
// Which machine will use it. Given, the credential is delivered by the mesh rather than
// printed for somebody to carry — which is the difference between the builder being a module
// and being a program somebody configures.
forNode := set.String("node", "",
"the machine that will run it, so the mesh delivers the credential instead of printing it")
module := set.String("module", "builder", "the module on that machine that will read it")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 2 || positionals[0] != "issue" {
return errors.New("builder issue <name> [--node <machine>]")
}
name := positionals[1]
management, err := broker.ManagementFromEnvironment()
if err != nil {
return err
}
// The same shape of secret a token carries: enough entropy that guessing is not a strategy,
// and safe to put in a URL because that is where it goes.
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return err
}
password := base64.RawURLEncoding.EncodeToString(raw)
if err := management.CreateBuilderAccount(ctx, name, password); err != nil {
return err
}
fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n",
name, link.BuildQueue, link.Exchange)
if *forNode != "" {
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
}
inv, err := openInventory(ctx)
if err != nil {
return err
}
defer inv.Close()
// The URL and what verifies the broker, together. A mesh's broker presents a certificate
// of the mesh's own, which is in no public trust store — so a URL on its own reaches only
// a broker somebody else vouches for, and the connection fails at TLS with an error about
// an unknown authority rather than about a missing pin.
//
// **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same
// way: out of band relative to the broker, so what is trusted does not come from the thing
// being trusted.
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
}{
URL: fmt.Sprintf("amqps://%s:%s@%s/", name, password, known.Address),
Fingerprint: known.Fingerprint,
})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil {
return err
}
// Not printed. It is sealed to that machine and the mesh cannot read it back, which is
// the whole point — printing it here would put the one copy that matters on a terminal.
fmt.Printf(" sealed to %s, for the %s module. It arrives with the next push.\n",
*forNode, *module)
fmt.Printf(" run `push %s` to send it\n", *forNode)
return nil
}
// The whole line only when the address is known. A URL with a placeholder where the host
// should be is a URL somebody pastes and then debugs, and the placeholder is the last thing
// they look at.
if known, err := broker.FromEnvironment(); err == nil {
fmt.Printf(" MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address)
} else {
fmt.Printf(" the password is %s\n\n", password)
fmt.Printf(" This control plane has no %s, so it cannot say where the broker is.\n"+
" Put the password in MESH_BROKER_AMQP on the build machine.\n\n",
broker.AddressVar)
}
// Shown once, like a token, and for the same reason: what is stored is the broker's own hash
// of it, and a control plane that could show it back would be a control plane that holds it.
fmt.Println("This is the only time it is shown.")
return nil
}
// buildBehind builds every module the mesh holds older than its source has.
//
// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
// records where each module came from and what its source last had; until this, a person read
// that list and retyped each repository — which is a person being the loop, and the thing a
// pipeline was doing before it was taken away.
//
// Each is built and recorded on its own. **One failing does not stop the others**, for the same
// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
// repository holds back nine good ones is a mesh where nobody dares add the tenth.
func buildBehind(ctx context.Context, wait time.Duration) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
held, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var stale []inventory.Entry
for _, e := range held {
if !e.Source.Current() {
stale = append(stale, e)
}
}
if len(stale) == 0 {
// Said rather than doing nothing quietly: "nothing needed building" and "this did not
// run" must never look the same.
fmt.Println("every module the mesh holds is what its source last had")
return nil
}
fmt.Printf("%d module(s) behind their source:\n", len(stale))
for _, e := range stale {
fmt.Printf(" %s %s < %s\n",
e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
}
fmt.Println()
var failed []string
for _, e := range stale {
fmt.Printf("--- %s\n", e.Manifest.Module)
// Its own recorded ref, not its head commit: a module tracking a branch should be built
// from that branch, and pinning to the commit the mesh happened to notice would quietly
// turn a tracked branch into a pin.
if err := buildOne(ctx, e.Source.Repository, e.Source.Path, e.Source.Ref, wait); err != nil {
fmt.Printf(" %v\n", err)
failed = append(failed, e.Manifest.Module)
}
}
if len(failed) > 0 {
return fmt.Errorf("%d of %d could not be built: %s",
len(failed), len(stale), strings.Join(failed, ", "))
}
fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
len(stale))
return nil
}
// buildOne asks a build machine for one repository and records everything that came back.
//
// Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, repository, path, ref string, wait time.Duration) error {
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
// Correlated by something the control plane makes, not by the module's name: two builds of one
// module can be in flight, and the second answer is not the first one's.
request := link.BuildRequest{
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
Repository: repository,
Path: path,
Ref: ref,
Held: heldBy(ctx),
}
fmt.Printf("asked for %s", request.Repository)
if path != "" {
fmt.Printf(" at %s", path)
}
if ref != "" {
fmt.Printf(" on %s", ref)
}
fmt.Println()
result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
if err != nil {
return err
}
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
// nobody asked for, and the difference is the whole of whether somebody should be looking at
// something.
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
return err
}
if result.Failed != "" {
// The builder's own words. Wrapping them in something about the control plane would put
// two explanations between a person and a build log.
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
}
for _, made := range result.Made {
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
}
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
// second thing to disagree about what a manifest is.
manifest, err := catalogue.ParseManifest(result.Manifest)
if err != nil {
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
}
// Recorded with where it came from, so "is this current?" is answerable without building it
// again (novox/hq ADR 0009).
if err := inv.RegisterModule(ctx, manifest, inventory.Source{
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
BuiltFrom: result.Commit, Head: result.Commit,
}); err != nil {
return err
}
fmt.Printf("\n%s %s, built on %s from %s\n",
manifest.Module, manifest.Version, result.On, short(result.Commit))
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
return nil
}
// buildAndShow builds and prints the manifest without recording anything.
func buildAndShow(ctx context.Context, repository, path, ref string, wait time.Duration) error {
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
Repository: repository, Path: path, Ref: ref,
Held: heldBy(ctx),
}, wait)
if err != nil {
return err
}
if result.Failed != "" {
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
}
manifest, err := catalogue.ParseManifest(result.Manifest)
if err != nil {
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
}
body, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
// answers is what the three questions came back with, read once.
type answers struct {
wrong []inventory.Doing
nodes []inventory.Node
quiet []inventory.Node
behind map[string][]string
sources map[string]inventory.Source
// waiting is every machine not running what the mesh would send it.
waiting []inventory.Machine
// reported is every machine's last word beside when it was last sent a declaration — the
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
// recorded at send, not at apply).
reported []inventory.Reported
// refused is why a machine cannot be worked out at all, by name. A different thing from every
// other answer here: those are about a machine that was told something, and this is about one
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
// to compare against, so without this a wholly blocked mesh reads as a well one.
refused map[string]string
// network is why the private network could not be computed, when it could not. Almost always
// a consequence of the refusals above: a node that does not resolve is not on the network, and
// a mesh whose hub is that node has no hub.
network string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
//
// **A failure here is not a failure to build.** A module that names no base does not need this at
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
// than a build command refusing to start because a query did not run. So the store not opening is
// reported and the build goes ahead without it.
func heldBy(ctx context.Context) map[string]string {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
"base will be told that base is missing: %v\n", err)
return nil
}
defer open.Close()
held, err := open.inventory.Held(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
return nil
}
return held
}
cmd/mesh-controller/licence.go
+481
View File
@@ -0,0 +1,481 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"strings"
)
// licenceCommand is everything about model access the mesh holds.
//
// **A licence is a named thing and the name is the operator's** (novox/hq ADR 0024). *The personal
// account*, *the organisation's account* — those are names a person uses, and the mesh has to use
// them too, because the whole point is saying which one a given consumer uses.
func licenceCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New(
"licence add|list|use|release|key|manager|set-grant|refresh|submit-refresh|forget")
}
switch args[0] {
case "add":
return licenceAdd(ctx, args[1:])
case "list":
return licenceList(ctx)
case "use":
return licenceUse(ctx, args[1:], true)
case "release":
return licenceUse(ctx, args[1:], false)
case "key":
return licenceKey(ctx, args[1:])
case "manager":
return licenceManager(ctx, args[1:])
case "set-grant":
return licenceSetGrant(ctx, args[1:])
case "refresh":
return licenceRefresh(ctx, args[1:])
case "submit-refresh":
return licenceSubmitRefresh(ctx, args[1:])
case "forget":
return licenceForget(ctx, args[1:])
}
return fmt.Errorf(
"licence %q; it is add, list, use, release, key, manager, set-grant, refresh, "+
"submit-refresh or forget", args[0])
}
func licenceAdd(ctx context.Context, args []string) error {
set := flag.NewFlagSet("licence add", flag.ContinueOnError)
// What a consumer must know that is not secret — a base URL, a model name. Never the key.
serves := set.String("serves", "",
"JSON a consumer must know that is not secret, such as a base URL or a model")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 2 {
return errors.New(`licence add <vendor> <name> [--serves '{"model":"..."}']`)
}
vendor, name := positionals[0], positionals[1]
values := map[string]any{}
if strings.TrimSpace(*serves) != "" {
if err := json.Unmarshal([]byte(*serves), &values); err != nil {
return fmt.Errorf("--serves is not JSON: %w", err)
}
}
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
if err := held.Add(ctx, name, vendor, values); err != nil {
return err
}
fmt.Printf("%s (%s) recorded. Nothing uses it yet, and it has no key:\n"+
" licence use %s <node> <module>\n licence key %s\n", name, vendor, name, name)
return nil
}
func licenceList(ctx context.Context) error {
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
all, err := held.All(ctx)
if err != nil {
return err
}
if len(all) == 0 {
// Said, not printed as nothing: an empty list and a failed read must never look the same.
fmt.Println("this mesh holds no licences")
return nil
}
for _, one := range all {
holders, err := held.HoldersOf(ctx, one.Name)
if err != nil {
return err
}
fmt.Printf("%s (%s)\n", one.Name, one.Vendor)
if len(holders) == 0 {
fmt.Printf(" nobody uses it\n")
}
for _, h := range holders {
// Whether it has a key is the question somebody is actually asking, so it is said
// per holder rather than per licence: the key was sealed to the holders that existed
// when it was supplied, and one recorded afterwards has none.
state := "has no key — supply it again with `licence key " + one.Name + "`"
if h.Sealed != "" {
state = "has a key"
}
fmt.Printf(" %s on %s: %s\n", h.Module, h.Node, state)
}
}
return nil
}
func licenceUse(ctx context.Context, args []string, using bool) error {
verb := "use"
if !using {
verb = "release"
}
if len(args) != 3 {
return fmt.Errorf("licence %s <name> <node> <module>", verb)
}
name, node, module := args[0], args[1], args[2]
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
if !using {
if err := held.StopUsing(ctx, name, node, module); err != nil {
return err
}
fmt.Printf("%s on %s no longer uses %s. Its copy of the key goes on the next push\n",
module, node, name)
return nil
}
if err := held.Use(ctx, name, node, module); err != nil {
return err
}
fmt.Printf("%s on %s uses %s.\n", module, node, name)
// The consequence, said now rather than discovered as a machine that resolves and receives
// nothing: the mesh discarded the plaintext, so a holder added after the key was supplied has
// no key and the mesh cannot make one.
sealed, err := held.KeyFor(ctx, name, node, module)
if err != nil {
return err
}
if sealed == "" {
fmt.Printf(" It has no key yet — the mesh discarded the plaintext when it was supplied "+
"and cannot seal another. Supply it again:\n licence key %s\n", name)
}
return nil
}
// licenceKey is the *accept* verb novox/hq ADR 0024 names as missing.
//
// Take a value, seal it to each holder, and discard the plaintext. Every other credential the
// mesh handles it generated itself; an API key arrives from a person, and a mesh that kept
// operator-supplied keys readably is the arrangement this project measured and rejected.
func licenceKey(ctx context.Context, args []string) error {
set := flag.NewFlagSet("licence key", flag.ContinueOnError)
// A file rather than an argument, by default. A key on a command line is a key in shell
// history and in every process listing taken while it ran.
from := set.String("file", "", "read the key from a file instead of standard input")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("licence key <name> [--file <path>]")
}
name := positionals[0]
var value string
if *from != "" {
raw, err := os.ReadFile(*from)
if err != nil {
return err
}
value = strings.TrimSpace(string(raw))
} else {
fmt.Fprintln(os.Stderr, "reading the key from standard input; it is not echoed anywhere")
reader := bufio.NewReader(os.Stdin)
line, err := reader.ReadString('\n')
if err != nil && line == "" {
return fmt.Errorf("nothing was given on standard input: %w", err)
}
value = strings.TrimSpace(line)
}
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
sealed, err := held.Accept(ctx, name, value, func(node string) (string, error) {
return inv.SealingKeyOf(ctx, node)
})
if err != nil {
if sealed > 0 {
// Some holders got it and some did not, and the person holding the key is the only
// one who can finish the job. Saying how far it got is the difference between running
// this again knowing what it will do and running it hoping.
return fmt.Errorf(
"%w\n\n%d holder(s) were sealed before this. Running `licence key %s` again "+
"with the same key seals the rest and changes nothing for those already done",
err, sealed, name)
}
return err
}
// Not echoed back, ever. What is stored is unreadable by whoever holds it, the mesh included,
// and printing the value here would put the one copy that matters on a terminal.
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
sealed)
fmt.Printf(" run `push` to deliver it\n")
return nil
}
// licenceManager names the one node that holds a refreshable-grant licence's refresh token readably
// and refreshes it centrally (novox/hq ADR 0050).
//
// **Only a refreshable-grant licence has one.** A static-key licence has no refresh token to hold, so
// naming a manager for it is refused where the mistake is made rather than kept as a field that means
// nothing — the absent manager is part of what keeps a static key from ever holding a value readably
// at rest.
func licenceManager(ctx context.Context, args []string) error {
if len(args) != 3 {
return errors.New("licence manager <name> <node> <module>")
}
name, node, module := args[0], args[1], args[2]
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
if err := held.SetManager(ctx, name, node, module); err != nil {
return err
}
fmt.Printf("%s on %s holds and refreshes %s.\n"+
" Its refresh token is sealed to %s's key — readable by that node alone, not by any other "+
"node and not by this database. Put %s on the licence too so it is delivered the token:\n"+
" licence use %s %s %s\n", module, node, name, node, module, name, node, module)
return nil
}
// sealedGrantJSON is the wire shape of a sealed refresh token on this command surface: an anonymous
// sealed box and the public key it was sealed to, and nothing else.
//
// **Every field of it is ciphertext or a public key.** `sealed` is the refresh token as a
// `crypto_box_seal` to the manager node's public key; `manager_key` is that public key. Neither is
// the refresh token in the clear — which is why this surface may read one in (`set-grant`,
// `submit-refresh`) without the control plane ever holding a refresh token it could read. The manager
// module, on the manager node, seals it; the HOST, on that node, unseals it to deliver cleartext. This
// database, and this surface, only ever forward the box (novox/hq ADR 0050).
type sealedGrantJSON struct {
Sealed string `json:"sealed"`
ManagerKey string `json:"manager_key"`
}
// readSealedGrant reads a sealed refresh token from a file or standard input as JSON.
func readSealedGrant(from string) (sealedGrantJSON, error) {
var raw []byte
var err error
if from != "" {
raw, err = os.ReadFile(from)
} else {
raw, err = readAllStdin()
}
if err != nil {
return sealedGrantJSON{}, err
}
var g sealedGrantJSON
if err := json.Unmarshal(raw, &g); err != nil {
return sealedGrantJSON{}, fmt.Errorf("the sealed refresh token is not JSON: %w", err)
}
if g.Sealed == "" || g.ManagerKey == "" {
return sealedGrantJSON{}, errors.New(
"a sealed refresh token is {sealed, manager_key}, and one part is missing")
}
return g, nil
}
func readAllStdin() ([]byte, error) {
reader := bufio.NewReader(os.Stdin)
return io.ReadAll(reader)
}
// licenceSetGrant stores a sealed refresh token the manager module produced — adoption, and the
// re-seal after a rotation done outside this process (novox/hq ADR 0050).
//
// **It takes a sealed box, never a refresh token.** The manager module, on the manager node, reads
// the operator's refresh token, seals it to that node's own public key, and hands the box here. So the
// one moment a refresh token is in the clear is on the manager node, never in the control plane — the
// same bound the whole carve-out keeps. This surface refuses anything that is not a complete sealed
// grant rather than storing half of one.
func licenceSetGrant(ctx context.Context, args []string) error {
set := flag.NewFlagSet("licence set-grant", flag.ContinueOnError)
from := set.String("file", "", "read the sealed refresh token from a file instead of standard input")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("licence set-grant <name> [--file <path>]")
}
name := positionals[0]
grant, err := readSealedGrant(*from)
if err != nil {
return err
}
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
if err := held.SetRefreshGrant(ctx, name, grant.Sealed, grant.ManagerKey); err != nil {
return err
}
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n",
"the manager", name)
return nil
}
// licenceSubmitRefresh publishes a refresh a MANAGER NODE already performed: the new access token is
// sealed to every holder, and a rotated refresh token replaces the stored envelope (novox/hq ADR
// 0050, Phase C).
//
// **This is the boundary the invariant rests on.** The manager runtime, on the manager node, opened
// the at-rest envelope with that node's key, called the vendor's OAuth endpoint, and produced this:
// the new access token in the clear, and — only if the vendor rotated it — the refresh token already
// re-sealed at rest. This reads exactly those two things and no refresh token in the clear ever
// reaches it, because it is never given one. The access token is sealed per holder and discarded,
// as any accepted key is; the rotated envelope is stored opaque.
func licenceSubmitRefresh(ctx context.Context, args []string) error {
set := flag.NewFlagSet("licence submit-refresh", flag.ContinueOnError)
accessFrom := set.String("access-file", "",
"read the new access token from a file instead of standard input")
grantFrom := set.String("grant-file", "",
"the rotated sealed refresh token, if the vendor rotated it; omit if it did not")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New(
"licence submit-refresh <name> [--access-file <path>] [--grant-file <path>]")
}
name := positionals[0]
var accessToken string
if *accessFrom != "" {
raw, err := os.ReadFile(*accessFrom)
if err != nil {
return err
}
accessToken = strings.TrimSpace(string(raw))
} else {
raw, err := readAllStdin()
if err != nil {
return err
}
accessToken = strings.TrimSpace(string(raw))
}
if accessToken == "" {
return errors.New("no access token was given, so there is nothing to seal")
}
// The rotated sealed token is optional: absent, the stored refresh token is left exactly as it was.
var newSealed, newManagerKey string
if *grantFrom != "" {
grant, err := readSealedGrant(*grantFrom)
if err != nil {
return err
}
newSealed, newManagerKey = grant.Sealed, grant.ManagerKey
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
held, err := open.Licences(ctx)
if err != nil {
return err
}
inv := open.inventory
sealed, err := held.SubmitRefresh(ctx, name, accessToken, newSealed, newManagerKey,
func(node string) (string, error) {
return inv.SealingKeyOf(ctx, node)
})
if err != nil {
return err
}
rotatedNote := "the refresh token was left with its manager unchanged"
if newSealed != "" {
rotatedNote = "the rotated refresh token replaced the stored box, still readable by the " +
"manager node alone"
}
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
" run `push` to deliver it\n", name, sealed, rotatedNote)
return nil
}
// licenceRefresh mints a new access token for a refreshable-grant licence and seals it to every
// holder (novox/hq ADR 0050). The refresh token stays with the manager and is never delivered.
//
// The vendor's actual refresh is a plug-in this build does not ship (Phase C), so here this reports
// that plainly rather than pretending to have refreshed.
func licenceRefresh(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("licence refresh <name>")
}
name := args[0]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
held, err := open.Licences(ctx)
if err != nil {
return err
}
inv := open.inventory
sealed, err := held.Refresh(ctx, name, func(node string) (string, error) {
return inv.SealingKeyOf(ctx, node)
})
if err != nil {
return err
}
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
"with its manager.\n run `push` to deliver it\n", name, sealed)
return nil
}
func licenceForget(ctx context.Context, args []string) error {
if len(args) != 1 {
return errors.New("licence forget <name>")
}
held, err := openLicences(ctx)
if err != nil {
return err
}
defer held.Close()
if err := held.Forget(ctx, args[0]); err != nil {
return err
}
// Said plainly, because the mesh cannot do it and pretending otherwise is worse than useless:
// a licence outliving its holder is a live credential nobody is watching.
fmt.Printf("%s is forgotten, and every record of who held it with it.\n"+
" The key itself is not the mesh's to revoke — do that where the licence was bought\n",
args[0])
return nil
}
cmd/mesh-controller/main.go
+208
View File
@@ -0,0 +1,208 @@
// Command mesh-controller is the control plane: everything that needs to know about more than one
// node (novox/hq ADR 0006).
//
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without.
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/store"
)
// version is stamped at link time. Unset in a development build, and it says so rather than
// claiming a number.
var version = "development build"
// held is a context this process was granted, and the schema it carries.
//
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
// yet, not because they are optional.
var held = []struct {
name string
migrations func() ([]store.Migration, error)
}{
{inventory.Name, inventory.Migrations},
{identity.Name, identity.Migrations},
{licences.Name, licences.Migrations},
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err)
os.Exit(1)
}
}
func run() error {
args := os.Args[1:]
if len(args) == 0 {
usage()
return fmt.Errorf("no command given")
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
switch args[0] {
case "build":
return buildCommand(ctx, args[1:])
case "builder":
return builderCommand(ctx, args[1:])
case "board":
return boardCommand(ctx, args[1:])
case "api":
return apiCommand(ctx, args[1:])
case "licence":
return licenceCommand(ctx, args[1:])
case "rotate":
return rotateCommand(ctx, args[1:])
case "builds":
return buildsCommand(ctx, args[1:])
case "pin":
return pinCommand(ctx, args[1:], true)
case "unpin":
return pinCommand(ctx, args[1:], false)
case "migrate":
return migrate(ctx)
case "node":
return nodeCommand(ctx, args[1:])
case "token":
return tokenCommand(ctx, args[1:])
case "identity":
return identityCommand(ctx, args[1:])
case "broker":
return brokerCommand(args[1:])
case "serve":
return serve(ctx)
case "upgrade":
return upgradeCommand(ctx, args[1:])
case "declare":
return declare(ctx, args[1:])
case "overlay":
return overlayCommand(ctx, args[1:])
case "module":
return moduleCommand(ctx, args[1:])
case "assign", "unassign":
return assignCommand(ctx, args[0], args[1:])
case "settings":
return settingsCommand(ctx, args[1:])
case "secret":
return secretCommand(ctx, args[1:])
case "plan":
return planCommand(ctx, args[1:])
case "push":
return pushCommand(ctx, args[1:])
case "status":
return statusCommand(ctx, args[1:])
case "version":
fmt.Println(version)
return nil
case "help", "-h", "--help":
usage()
return nil
default:
usage()
return fmt.Errorf("%q is not a command", args[0])
}
}
func usage() {
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
migrate bring each context's schema up to date
node add <name> create a node record
node list the nodes this mesh knows about
node show <name> what one machine reported it can do, and why
node public-domain <name> the domain it composes its routed names under
node public-domain <name> <d> ...set it to d
node public-domain <name> --clear ...it faces the outside no longer
token issue --node <name> a one-time right to join, for an existing record
token issue --new <name> create the record and issue for it
identity show this control plane's signing key
broker show where the broker is, and what to expect there
serve consume what nodes say, and answer
declare <node> <file> send a node a signed declaration
overlay place <node> [flags] say where a node is and how it is reached
overlay show the private network, as the mesh computes it
module add <file> register a module from its manifest
module list what modules this mesh knows about
module moved <name> <commit> the source has a newer commit than the mesh built
module forget <name> remove one, unless a node runs it or the mesh holds things for it
module forget <name> --and-what-it-holds ...and discard its settings, secrets and ports too
module issue <name> --node <m> a broker account for a module, scoped to its emits and consumes
upgrade <name> what happens when this module's current version moves
upgrade <name> roll-out [--together] ...send it to the machines running it
upgrade <name> record ...record that they are behind, and send nothing
status [--json] what is wrong, what is quiet, and what is out of date
board [--listen ADDR] the same three questions, as a page that holds nothing
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
assign <node> <module> put a module on a node
unassign <node> <module> take it off
settings set <module> <file> what a module's config should say, for the whole mesh
settings set <module> <file> --node <n> ...or for one machine
settings clear <module> [--node <n>] take a layer away
secret accept <node> <module> <name> carry a value the mesh did not make and cannot invent
secret accept ... --from <file> ...read it from a file rather than being asked
build <repository> [--ref R] have a build machine build it, and record what came out
build --behind build every module the mesh holds older than its source
builds [<module>] what has been built lately, and what came of it
builder issue <name> a broker account for a build machine, scoped to build work
licence add|list|use|key model access, under the name a person calls it
licence manager <name> <node> the node that holds a refreshable licence's refresh token
licence refresh <name> mint a new access token and seal it to every holder
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
pin <node> <provision> <from> which node this one gets a provision from
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
push [<node>] [--behind] send a node everything it should be, or only those that need it
version what this binary is
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
`+store.Variable("<context>")+` — or `+store.FileVariable("<context>")+`, naming a file that holds
the same thing and keeps the password out of the environment. This process holds:
`)
for _, c := range held {
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
c.name, store.Database(c.name), store.Variable(c.name))
}
fmt.Fprintln(os.Stderr)
}
// parseAround reads flags that may sit before, after or between positional arguments.
//
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
// parses no flags at all and silently ignores every one of them. The host learned this the same
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
// keeps naming, and it looks exactly like success.
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
var positionals []string
rest := args
for {
if err := set.Parse(rest); err != nil {
return nil, err
}
rest = set.Args()
if len(rest) == 0 {
return positionals, nil
}
positionals = append(positionals, rest[0])
rest = rest[1:]
}
}
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
return b.inv.RecordBuild(ctx, buildFrom(result))
}
cmd/mesh-controller/mesh_for_test.go
+108
View File
@@ -0,0 +1,108 @@
package main
import (
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/overlay"
)
// A mesh a command can be run against.
//
// The commands here were tested through the pieces they call and never through themselves, so
// three faults that only exist where the pieces meet — an assignment reported as fine while it
// blocked other machines, a read-shaped invocation that wrote, a JSON interface that stopped
// emitting JSON — were invisible to every test in this package. This raises the real stores and
// calls the real functions.
// aMesh is two placed, capable machines on a private network, with nothing assigned but the
// network itself.
//
// `anchor` is the hub. That is not decoration: a mesh whose hub cannot be resolved has no private
// network at all, which is how one machine's problem reaches every other.
func aMesh(t *testing.T) *stores {
t.Helper()
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
licences.ForTest(t) // planning reaches this one too, by name and never by connection
open, err := openStores(t.Context())
if err != nil {
t.Fatal(err)
}
t.Cleanup(open.Close)
for _, m := range provided {
if err := open.inventory.Provide(t.Context(), m); err != nil {
t.Fatal(err)
}
}
for i, name := range []string{"anchor", "laptop"} {
record, err := open.inventory.AddNode(t.Context(), name)
if err != nil {
t.Fatal(err)
}
if err := open.inventory.SetPlace(t.Context(), name, name+".example:51820", "here",
name == "anchor", fmt.Sprintf("10.77.0.%d", i+1)); err != nil {
t.Fatal(err)
}
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
{"name": "container-runtime", "present": true},
{"name": "wireguard", "present": true},
{"name": "systemd", "present": true},
}})
if err != nil {
t.Fatal(err)
}
var profile map[string]any
if err := json.Unmarshal(reported, &profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
t.Fatal(err)
}
}
return open
}
// aPublicKey is a key a machine could have reported. Its private half is thrown away: nothing here
// opens anything, it only needs the mesh to believe a machine has a key.
func aPublicKey(t *testing.T) string {
t.Helper()
k, err := ecdh.X25519().GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
}
// register puts a manifest in the catalogue.
func register(t *testing.T, open *stores, m catalogue.Manifest) {
t.Helper()
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{}); err != nil {
t.Fatal(err)
}
}
// rivals are two modules that cannot share a machine, which is the shortest way to make a node's
// own set of assignments incoherent using nothing but commands a person has.
func rivals() (catalogue.Manifest, catalogue.Manifest) {
claim := []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
}
cmd/mesh-controller/modules.go
+463
View File
@@ -0,0 +1,463 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// the catalogue: what exists, what is assigned, and how it is configured.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// provided is what comes with the control plane rather than from a repository.
//
// WireGuard, the names, and the domain module over both. The first two are here because the code
// that works out their files is here:
// a peer list is derived from every machine at once, so it cannot be written in a manifest, and
// whatever computes it has to live wherever the whole picture is.
//
// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent
// from a machine nobody gave it to.
func providedModules() []catalogue.Manifest {
var out []catalogue.Manifest
for _, raw := range []map[string]any{
// **Two used to be here and are gone**: one wrote the mesh's names into a hosts file, the
// other wrote the same machines as wildcards for a resolver to read. Neither ran software
// and neither could be swapped for anything, which is the test of whether a thing is a
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
// to live, and now a module says where it wants it — `facts` in its own manifest.
overlay.Manifest(), overlay.DomainManifest(),
} {
var m catalogue.Manifest
b, _ := json.Marshal(raw)
_ = json.Unmarshal(b, &m)
out = append(out, m)
}
return out
}
var provided = providedModules()
func moduleCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("module add <file>, module list, or module forget <name>")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
switch args[0] {
case "add":
set := flag.NewFlagSet("module add", flag.ContinueOnError)
repo := set.String("source", "", "where this module comes from")
ref := set.String("ref", "", "the branch followed there")
commit := set.String("commit", "", "the commit this manifest was read at")
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("module add <manifest.json> [--source <repo> --ref <branch> --commit <sha>]")
}
raw, err := os.ReadFile(positionals[0])
if err != nil {
return err
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
return err
}
// Provenance together or not at all. A source with no commit cannot be compared against
// anything, so it would record where the module came from and still never be able to say
// the mesh is behind it — which is the one thing recording it is for.
if (*repo == "") != (*commit == "") {
return errors.New("--source and --commit go together: a source with no commit " +
"cannot be compared against anything, and a commit with no source has nothing " +
"to be compared with")
}
if err := inv.RegisterModule(ctx, m, inventory.Source{
Repository: *repo, Ref: *ref, BuiltFrom: *commit,
}); err != nil {
return err
}
fmt.Printf("%s registered", m.Module)
if *commit != "" {
fmt.Printf(" from %s", short(*commit))
}
if len(m.Provides) > 0 {
fmt.Printf(", providing %s", describeOffers(m.Provides))
}
fmt.Println()
for _, c := range m.Claims {
fmt.Printf(" claims %s, one per %s\n", c.Name, c.At())
}
return nil
case "list":
// The catalogue: what exists, where it came from, whether it is current, and who runs it.
// The provenance was recorded from the first build and nothing showed it, which made
// "is this current?" a question you could only answer by reading the database.
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
if len(entries) == 0 {
fmt.Println("this mesh knows about no modules yet")
return nil
}
var stale int
for _, e := range entries {
m := e.Manifest
fmt.Printf("%-18s %-8s", m.Module, m.Version)
switch {
case e.Provided:
fmt.Printf(" %-22s", "with the control plane")
case e.Source.Repository == "":
// Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and
// worth saying, because nothing can rebuild it.
fmt.Printf(" %-22s", "handed over")
case !e.Source.Current():
stale++
fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head))
default:
fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom))
}
if len(e.On) > 0 {
fmt.Printf(" on %s", strings.Join(e.On, ", "))
} else {
fmt.Printf(" on nothing")
}
fmt.Println()
var says []string
if len(m.Provides) > 0 {
says = append(says, "provides "+describeOffers(m.Provides))
}
if len(m.Requires) > 0 {
says = append(says, "requires "+strings.Join(m.Requires, ", "))
}
for _, c := range m.Claims {
says = append(says, "claims "+c.At()+"/"+c.Name)
}
if len(m.Capabilities) > 0 {
says = append(says, "needs "+strings.Join(m.Capabilities, ", "))
}
if len(says) > 0 {
fmt.Printf(" %s\n", strings.Join(says, " · "))
}
}
if stale > 0 {
fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale)
}
return nil
case "moved":
if len(args) != 3 {
return errors.New("module moved <name> <commit> — the source has a newer commit")
}
if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil {
return err
}
from, err := inv.SourceOf(ctx, args[1])
if err != nil {
return err
}
if from.Current() {
fmt.Printf("%s is current at %s\n", args[1], short(from.Head))
return nil
}
fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
args[1], short(from.BuiltFrom), short(from.Head))
fmt.Printf(" run `build %s` to catch up\n", from.Repository)
return nil
case "forget":
// **What goes with it is said before it goes** (novox/hq 04-ISSUES/017). The settings, the
// module's own secrets and the ports the mesh chose all cascade off the module row, so
// `forget` used to destroy them and report "forgotten" — an action succeeding into a state
// its own verify would reject, and a sealed secret is not recoverable afterwards.
set := flag.NewFlagSet("module forget", flag.ContinueOnError)
andHeld := set.Bool("and-what-it-holds", false,
"discard its settings, its own secrets and its ports along with it")
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("module forget <name> [--and-what-it-holds]")
}
if !*andHeld {
if err := inv.ForgetModule(ctx, positionals[0]); err != nil {
return err
}
fmt.Printf("%s forgotten\n", positionals[0])
return nil
}
held, err := inv.DiscardModule(ctx, positionals[0])
if err != nil {
return err
}
fmt.Printf("%s forgotten\n", positionals[0])
for _, line := range held.Lines() {
// Said after the fact as well as before it: this is the only record that these
// existed, and the next person to ask why the module came back empty reads it here.
fmt.Printf(" discarded%s\n", strings.TrimPrefix(line, " "))
}
return nil
case "issue":
// A module's broker account, scoped by its emits and consumes (novox/hq ADR 0043) and
// sealed to the machine that will run it — the generic case the builder was the first of.
set := flag.NewFlagSet("module issue", flag.ContinueOnError)
forNode := set.String("node", "",
"the machine that will run it, so the credential is delivered instead of printed")
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("module issue <module> --node <machine>")
}
module := positionals[0]
if *forNode == "" {
return errors.New("module issue needs --node: a module's account is sealed to the " +
"machine that runs it, and the mesh cannot read it back to print")
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
m, ok := shelf[module]
if !ok {
return fmt.Errorf("this mesh knows no module %q; `module add` it first", module)
}
management, err := broker.ManagementFromEnvironment()
if err != nil {
return err
}
// The foundation owns the bus; make sure it exists before a module binds onto it.
if err := management.EnsureEventExchanges(ctx); err != nil {
return err
}
secret := make([]byte, 32)
if _, err := rand.Read(secret); err != nil {
return err
}
password := base64.RawURLEncoding.EncodeToString(secret)
account, err := management.CreateModuleAccount(ctx, *forNode, module, password, m.Emits, m.Consumes)
if err != nil {
return err
}
// A consumer's queue, with its dead-letter, is the foundation's to declare — its own account
// may not (ADR 0043). Made now, so it exists before the module binds onto it.
if len(m.Consumes) > 0 {
if err := management.EnsureModuleQueue(ctx, *forNode, module); err != nil {
return err
}
}
known, err := broker.FromEnvironment()
if err != nil {
return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
}
// The URL and what verifies the broker, together — a mesh's broker presents its own
// certificate, in no public trust store, so a URL alone fails at TLS (as `builder issue`).
held, err := json.Marshal(struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node"`
Module string `json:"module"`
}{
URL: fmt.Sprintf("amqps://%s:%s@%s/", account, password, known.Address),
Fingerprint: known.Fingerprint,
// The node and module the account is for, so the runtime names its queue as the mesh
// scoped it (<node>.<module>.events) without a manifest having to interpolate a node.
Node: *forNode,
Module: module,
})
if err != nil {
return err
}
if err := inv.AcceptSecretForModule(ctx, *forNode, module, "broker", string(held)); err != nil {
return err
}
fmt.Printf("broker account %s created for %s, scoped to what it emits and consumes\n",
account, module)
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n",
*forNode, *forNode)
return nil
default:
return fmt.Errorf("module has no %q; it has add, list, moved, forget and issue", args[0])
}
}
func assignCommand(ctx context.Context, verb string, args []string) error {
if len(args) != 2 {
return fmt.Errorf("%s <node> <module>", verb)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
// The act itself is in acts.go, so the command API refuses exactly what this refuses
// (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed.
act := assign
if verb == "unassign" {
act = unassign
}
said, err := act(ctx, open, args[0], args[1])
if said != "" {
fmt.Println(said)
}
if err != nil {
fmt.Println()
return err
}
return nil
}
func settingsCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
set := flag.NewFlagSet("settings", flag.ContinueOnError)
node := set.String("node", "", "one machine, rather than the whole mesh")
positionals, err := parseAround(set, args[1:])
if err != nil {
return err
}
where := "the whole mesh"
if *node != "" {
where = *node
}
switch args[0] {
case "set":
if len(positionals) != 2 {
return errors.New("settings set <module> <settings.json> [--node <node>]")
}
raw, err := os.ReadFile(positionals[1])
if err != nil {
return err
}
var values map[string]any
if err := json.Unmarshal(raw, &values); err != nil {
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
}
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
return err
}
var keys []string
for k := range values {
keys = append(keys, k)
}
sort.Strings(keys)
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
fmt.Println(" run `push` to send it")
return nil
case "clear":
if len(positionals) != 1 {
return errors.New("settings clear <module> [--node <node>]")
}
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
return err
}
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
return nil
default:
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
}
}
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
// entirely different things about where the answer has to be.
func describeOffers(offers []catalogue.Offer) string {
var out []string
for _, o := range offers {
if o.At() == catalogue.ScopeMesh {
out = append(out, o.Name+" (from anywhere in the mesh)")
continue
}
out = append(out, o.Name)
}
return strings.Join(out, ", ")
}
// pinCommand says which node a machine gets a provision from.
//
// Needed only when more than one could answer, and recordable before that -- a mesh with one
// database should not change where an existing machine gets its data the day a second one
// arrives.
func pinCommand(ctx context.Context, args []string, setting bool) error {
if setting && len(args) != 3 {
return errors.New("pin <node> <provision> <from-node>")
}
if !setting && len(args) != 2 {
return errors.New("unpin <node> <provision>")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
if !setting {
if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil {
return err
}
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
return nil
}
if args[0] == args[2] {
// Allowed by nothing here, and worth saying rather than resolving into a confusing
// refusal later: a node providing something to itself is a node-scoped provision, and
// this field is for the other kind.
return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+
"provides, which does not need saying", args[0], args[1])
}
if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil {
return err
}
fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2])
fmt.Printf(" run `push %s` to send it\n", args[0])
return nil
}
cmd/mesh-controller/network.go
+385
View File
@@ -0,0 +1,385 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// the private network: who is on it, where, and what they are called.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
func overlayCIDR() string {
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
return v
}
return "10.42.0.0/16"
}
func overlayCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("overlay place <node> [flags], or overlay show")
}
// Answered before anything is opened. A message about which command to use should not need a
// database to say so, and needing one turns a redirect into a connection error.
if args[0] == "push" {
return errors.New("`overlay push` is now `push`, which sends a node its network AND " +
"what its assignments resolve to — the two are computed from one picture of the " +
"mesh, and sending them separately would let them disagree")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
switch args[0] {
case "place":
return overlayPlace(ctx, inv, args[1:])
case "show":
return overlayShow(ctx, open)
default:
return fmt.Errorf("overlay has no %q; it has place and show", args[0])
}
}
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
if len(args) == 0 {
return errors.New(
"overlay place <node> [--endpoint host:port] [--site name] [--hub], or --nothing")
}
node := args[0]
set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
site := set.String("site", "", "where this machine physically is, or empty if it roams")
hub := set.Bool("hub", false, "this node is the hub every other routes through")
nothing := set.Bool("nothing", false,
"place it with nothing set: not dialable, no site, not the hub")
if err := set.Parse(args[1:]); err != nil {
return err
}
// **All three are declared together, so saying nothing took all three away.** The sibling of
// `node public-domain`: `overlay place anchor` reads like it places the node it names, and it
// silently unset the endpoint every other machine dials, the site it is in, and the hub if it
// was the hub — every path through it going with them, at the moment somebody was trying to
// look at it.
//
// A placement with nothing set is a real thing to want — a machine that roams and opens every
// path itself is exactly that — so it keeps a way to say so, by name.
if set.NFlag() == 0 {
return fmt.Errorf("overlay place %s was given nothing to place it with, and all three are "+
"declared together — it would take away the endpoint other machines dial %s at, its "+
"site, and the hub if it is the hub. Say --endpoint/--site/--hub, or --nothing if that "+
"is what you meant", node, node)
}
if *nothing && (*endpoint != "" || *site != "" || *hub) {
return fmt.Errorf("give %s a placement or --nothing, not both: they say opposite things "+
"and the mesh will not choose between them", node)
}
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
// election by address prefix fails silently (novox/hq ADR 0007).
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
return err
}
found, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
if err != nil {
return err
}
fmt.Printf("%s is at %s on the overlay\n", node, address)
switch {
case *hub:
fmt.Println(" the hub — every node not sharing a site routes through it")
case *endpoint == "":
fmt.Println(" not dialable — it opens every path itself")
}
if *site != "" {
fmt.Printf(" at %s, so it peers directly with anything else there\n", *site)
}
return nil
}
// network builds the private network over the machines that resolved the module for it.
//
// Not over every node the mesh knows. **A machine is on the private network because it was given
// the module**, and one that was not is absent from every peer list and from the names — which is
// the only thing "not on the network" can mean. Until this, having an address was enough, and
// there was no way to keep a machine off.
//
// Every node at once, which is the whole reason this is the control plane's work: a peer list is
// derived from all the others, so no node could compute its own.
func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
refused map[string]string) (*overlay.Generator, error) {
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
nodes := make([]overlay.Node, 0, len(places))
for _, p := range places {
if !on[p.Name] {
continue
}
nodes = append(nodes, overlay.Node{
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
Site: p.Site, Hub: p.Hub, Address: p.Address,
})
}
if len(nodes) == 0 {
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
// answers rather than refusing -- Compute would refuse for want of a hub, and reporting
// "no hub" to somebody who never asked for a network would be a lie about the cause.
return overlay.Empty(), nil
}
g, err := overlay.From(nodes, overlayCIDR(), "")
if err != nil && len(refused) > 0 {
// The network is missing something, and some machines could not be resolved at all. Those
// are almost always the same fact: a node that does not resolve contributes nothing, so
// reporting "no hub" would name a consequence and hide the cause.
var who []string
for name, why := range refused {
who = append(who, fmt.Sprintf(" %s: %s", name, why))
}
sort.Strings(who)
return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+
"probably why:\n%s", err, len(refused), strings.Join(who, "\n"))
}
return g, err
}
// graph is the whole mesh's network, for showing it.
func graph(ctx context.Context, open *stores) ([]overlay.Node, overlay.Graph, error) {
inv := open.inventory
on, refused, err := whoResolves(ctx, open, overlay.Requirement)
if err != nil {
return nil, nil, err
}
g, err := network(ctx, inv, on, refused)
if err != nil {
return nil, nil, err
}
return g.Nodes(), g.Graph(), nil
}
// whoResolves is the machines whose resolution answers a requirement, and why the others did not.
//
// By what a module **provides**, not by its name. WireGuard is one way to have a private network
// and there could be others, so a machine is on the network because something it runs provides
// one — asking for a particular module by name would be the mistake this whole mechanism exists
// to avoid.
//
// Resolved rather than read from the assignment table, because a module can arrive by being
// required by something else, and a machine that needs the private network to do its job is on it
// for the same reason as one that was handed it directly.
func whoResolves(ctx context.Context, open *stores, requirement string) (
map[string]bool, map[string]string, error) {
inv := open.inventory
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, nil, err
}
on := map[string]bool{}
// Why a node could not be resolved, kept rather than raised: one broken node must not stop
// the rest being described, and whoever is rendering that node will raise it themselves.
refused := map[string]string{}
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
if err != nil {
refused[n.Name] = err.Error()
continue
}
for _, m := range plan.Modules {
for _, offered := range m.Offers() {
if offered == requirement {
on[n.Name] = true
}
}
}
}
return on, refused, nil
}
// rendering is everything a declaration needs, computed over the whole mesh.
func generators(ctx context.Context, open *stores) (
map[string]catalogue.Generator, error) {
inv := open.inventory
on, refused, err := whoResolves(ctx, open, overlay.Addressing)
if err != nil {
return nil, err
}
net, err := network(ctx, inv, on, refused)
if err != nil {
return nil, err
}
// **One generator now.** Two more used to sit beside it — the names and a resolver's zone
// file — as modules that ran nothing. Both are facts a module asks for in its manifest
// (`facts:` — catalogue.FactsInto), computed from the same machines this sees: the ones on the
// private network, because a name for a machine not on it would resolve to an address nothing
// can reach.
return map[string]catalogue.Generator{
overlay.Name: net,
}, nil
}
func overlayShow(ctx context.Context, open *stores) error {
nodes, computed, err := graph(ctx, open)
if err != nil {
return err
}
if len(nodes) == 0 {
// Not "this mesh has no nodes", which it said until the network became a module and was
// then a lie about the cause: a mesh can have every node it will ever have and nobody on
// the private network, because nobody asked for one.
fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n",
overlay.Name)
return nil
}
for _, n := range nodes {
place := n.Address
if place == "" {
// Said, not skipped. A node with no place is a node with no network, and it should
// be visible here rather than quietly absent from a list of who is on it.
place = "no address — run `overlay place`"
}
fmt.Printf("%-16s %-14s", n.Name, place)
switch {
case n.Hub:
fmt.Print(" hub")
case !n.Reachable():
fmt.Print(" not dialable")
}
if n.Site != "" {
fmt.Printf(" at %s", n.Site)
}
fmt.Println()
for _, p := range computed[n.Name] {
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
}
}
return nil
}
// SilentFor is how long a node may be quiet before the mesh says so.
//
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
// is not the point — being able to say "out of touch" at all is, and nothing could before.
const SilentFor = 3 * time.Minute
// whereEveryoneIs is each machine's name on the private network, for the ones on it.
//
// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every
// other path here answers a question about one node by resolving the others; this one is called
// *from* that path, so doing the same would not terminate — which it did not, for two minutes,
// until it was run.
//
// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the
// private network depends on what it was assigned and what that requires, both of which are local
// facts. What it takes *from* other machines does not change the answer.
//
// The distinction that matters is kept: a machine absent from the network module's own view is
// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the
// network became something a machine is given.
func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest) (map[string]string, error) {
if shelf == nil {
// Refused rather than answered. Being on the private network is a conclusion about what a
// node resolves to, so with no catalogue nothing resolves and the honest answer is
// "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused
// every certificate the mesh was asked for while saying the machine was on no network.
return nil, errors.New(
"asked where everyone is without the catalogue, which cannot be answered")
}
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, p := range places {
if p.Address == "" {
continue
}
assigned, err := inv.Assigned(ctx, p.Name)
if err != nil || len(assigned) == 0 {
continue
}
caps, _ := inv.ProfileOf(ctx, p.Name)
got, err := catalogue.Resolve(shelf, assigned,
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
catalogue.World{Unchecked: true})
if err != nil {
continue
}
for _, m := range got.Modules {
for _, offered := range m.Offers() {
if offered == overlay.Requirement {
out[p.Name] = overlay.InternalName(p.Name)
}
}
}
}
return out, nil
}
// onThePrivateNetwork is every node's address on the overlay, sorted.
//
// A node with no address is left out rather than rendered as an empty source: an empty entry in a
// source set is a syntax error in the rule file, and a rule file that does not load leaves the
// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule,
// because nothing reports it.
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory) ([]string, error) {
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
var out []string
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
out = append(out, p.Address)
}
}
sort.Strings(out)
return out, nil
}
// namesInTheMesh is every machine's internal name and the address behind it.
//
// A machine with no address has no name: writing one that resolves to nothing is worse than not
// writing it, because a connection to an address that does not answer hangs where a name that
// does not resolve fails at once and says so. That is the rule the hosts file already follows,
// and this is the same set read the same way.
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) == "" {
continue
}
out[overlay.InternalName(p.Name)] = p.Address
}
return out, nil
}
cmd/mesh-controller/network_test.go
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"context"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// placementOf is what the mesh holds about where one node is.
func placementOf(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string) inventory.Overlay {
t.Helper()
placed, err := inv.Overlays(ctx)
if err != nil {
t.Fatal(err)
}
for _, one := range placed {
if one.Name == name {
return one
}
}
t.Fatalf("%s is not placed at all", name)
return inventory.Overlay{}
}
// The sibling of `node public-domain`, and the worse one: a placement is three facts declared
// together, so an invocation that said none of them took all three away — the endpoint every other
// machine dials, the site, and the hub. A mesh whose hub was placed that way has no paths left.
func TestPlacingANodeWithNothingSaidDoesNotUnplaceIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := overlayPlace(ctx, open.inventory,
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"}); err != nil {
t.Fatal(err)
}
err := overlayPlace(ctx, open.inventory, []string{"anchor"})
if err == nil {
t.Fatal("saying nothing unplaced the node instead of being refused")
}
if !strings.Contains(err.Error(), "--nothing") {
t.Errorf("the refusal does not say how to mean it: %v", err)
}
held := placementOf(t, ctx, open.inventory, "anchor")
if held.Endpoint != "198.51.100.10:51820" || held.Site != "hosting" || !held.Hub {
t.Fatalf("the placement was taken away by an invocation that was refused: %+v", held)
}
}
// Placing a machine with nothing set is a real thing to want — one that roams and opens every path
// itself is exactly that — so it keeps a way to be said, by name.
func TestPlacingANodeWithNothingIsAskedForByName(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := overlayPlace(ctx, open.inventory,
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting"}); err != nil {
t.Fatal(err)
}
if err := overlayPlace(ctx, open.inventory, []string{"anchor", "--nothing"}); err != nil {
t.Fatal(err)
}
held := placementOf(t, ctx, open.inventory, "anchor")
if held.Endpoint != "" || held.Site != "" || held.Hub {
t.Fatalf("--nothing did not place it with nothing: %+v", held)
}
}
// Both at once cannot be meant, so neither silently wins.
func TestAPlacementAndNothingTogetherIsRefused(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := overlayPlace(ctx, open.inventory,
[]string{"anchor", "--site", "hosting", "--nothing"}); err == nil {
t.Fatal("a placement and --nothing together was accepted")
}
}
cmd/mesh-controller/nodes.go
+383
View File
@@ -0,0 +1,383 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/token"
)
// what a machine is, and what it is allowed to be told.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
func nodeCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
switch args[0] {
case "show":
if len(args) != 2 {
return errors.New("node show <name>")
}
return showNode(ctx, inv, args[1])
case "add":
if len(args) != 2 {
return errors.New("node add <name>")
}
node, err := inv.AddNode(ctx, args[1])
if err != nil {
return err
}
fmt.Printf("added %s (%s)\n", node.Name, node.ID)
return nil
case "list":
nodes, err := inv.Nodes(ctx)
if err != nil {
return err
}
if len(nodes) == 0 {
// Said rather than printed as nothing: an empty list and a failed read must never
// look the same, and this command answering "none" is only honest because getting
// here means the store answered.
fmt.Println("this mesh has no node records yet")
return nil
}
for _, n := range nodes {
fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID)
}
return nil
case "public-domain":
// The domain this node composes its routed names under (novox/hq ADR 0066).
//
// **The form with no argument reports; clearing is asked for by name.** It used to clear —
// so `node public-domain anchor`, which reads like a question and is what anybody types to
// find out what the answer is, silently took every routed name the node had. A read-shaped
// invocation must never be a destructive write: there is no output that makes up for it,
// because the damage is already done by the time it prints.
return publicDomain(ctx, inv, args[1:])
default:
return fmt.Errorf("node has no %q; it has add, list, show and public-domain", args[0])
}
}
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
const publicDomainUsage = "node public-domain <name> — what it is now; " +
"<name> <domain> to set it; <name> --clear to take it away"
// publicDomain reads, sets or clears the domain a node composes its routed names under.
//
// Three forms, and the destructive one is the only one that has to be asked for. Clearing is a
// real thing to want — a machine that stops facing the outside composes no names, and
// lab-versus-production is this one setting (novox/hq ADR 0066) — so it keeps a way to say it.
// What it does not keep is being the thing that happens when nothing was said at all.
func publicDomain(ctx context.Context, inv *inventory.Inventory, args []string) error {
set := flag.NewFlagSet("node public-domain", flag.ContinueOnError)
clear := set.Bool("clear", false, "take the domain away; it composes no routed names after")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) == 0 || len(positionals) > 2 {
return errors.New(publicDomainUsage)
}
node := positionals[0]
switch {
case *clear && len(positionals) == 2:
// Both, which cannot be meant. Refused rather than one of them silently winning.
return fmt.Errorf("give %s a domain or --clear, not both: %q and --clear say opposite "+
"things and the mesh will not choose between them", node, positionals[1])
case *clear:
if err := inv.SetPublicDomain(ctx, node, ""); err != nil {
return err
}
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
fmt.Printf(" run `push %s` to take them off it\n", node)
return nil
case len(positionals) == 2:
if err := inv.SetPublicDomain(ctx, node, positionals[1]); err != nil {
return err
}
fmt.Printf("%s composes its routed names under %s\n", node, positionals[1])
fmt.Printf(" run `push %s` to send it\n", node)
return nil
default:
// Asked, so answered. NodeByName first, so a name the mesh has never heard of is a refusal
// rather than "it has no public domain", which is true of that name and says nothing.
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
domain, err := inv.PublicDomainOf(ctx, node)
if err != nil {
return err
}
if domain == "" {
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
fmt.Printf(" `node public-domain %s <domain>` gives it one\n", node)
return nil
}
fmt.Printf("%s composes its routed names under %s\n", node, domain)
return nil
}
}
func tokenCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "issue" {
return errors.New("token issue --node <name>, or token issue --new <name>")
}
set := flag.NewFlagSet("token issue", flag.ContinueOnError)
existing := set.String("node", "", "issue for a node record that already exists")
fresh := set.String("new", "", "create the node record, then issue for it")
validFor := set.Duration("for", time.Hour, "how long the token may be used")
if err := set.Parse(args[1:]); err != nil {
return err
}
// Exactly one, because the difference is what the token binds to. A command that guessed
// would sometimes create a second record for a machine that already has one.
if (*existing == "") == (*fresh == "") {
return errors.New("give exactly one of --node <name> or --new <name>: the first is a " +
"machine the mesh already has a record for, the second is one it has never seen")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
name := *existing
if *fresh != "" {
node, err := inv.AddNode(ctx, *fresh)
if err != nil {
return err
}
name = node.Name
}
issued, err := inv.IssueToken(ctx, name, *validFor)
if err != nil {
return err
}
// Assembled from two contexts by the process that holds both grants. Neither reads the
// other's store (novox/hq ADR 0008) — each is asked for its own part.
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
key, err := ident.Establish(ctx)
if err != nil {
return err
}
// The account is created before the token is handed over, which is what removes the
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
// exist before it does. The one-time secret IS the password, so a node's first connection is
// already authenticated and enrolment is what happens over it.
if management, err := broker.ManagementFromEnvironment(); err == nil {
if err := management.CreateNodeAccount(ctx, issued.Node.Name, issued.Secret); err != nil {
return err
}
fmt.Printf("broker account %s created, scoped to %s and the %s exchange\n\n",
issued.Node.Name, link.QueueFor(issued.Node.Name), link.Exchange)
} else if !errors.Is(err, broker.ErrNotConfigured) {
return err
}
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret}
// Absent is a state, not a failure: a control plane can hold records and a key before it has
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
known, err := broker.FromEnvironment()
switch {
case err == nil:
made.Broker, made.Fingerprint = known.Address, known.Fingerprint
case errors.Is(err, broker.ErrNotConfigured):
default:
return err
}
encoded, err := made.Encode()
if err != nil {
return err
}
fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n",
issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded)
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
if missing := made.Missing(); len(missing) > 0 {
fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n")
for _, m := range missing {
fmt.Printf(" - %s\n", m)
}
fmt.Printf("\nSet %s and %s once the broker is raised.\n",
broker.AddressVar, broker.CertificateVar)
}
return nil
}
func identityCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("identity show")
}
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
// Establish rather than read: a control plane asked for its identity before it has one should
// get one, not an error. Generating it is idempotent, so this is safe to run at any time.
key, err := ident.Establish(ctx)
if err != nil {
return err
}
fmt.Printf("signing key %s\n", key.ID)
fmt.Printf("fingerprint %s\n", key.Fingerprint())
fmt.Printf("created %s\n", key.Created.Format(time.RFC3339))
fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" +
"declaration because it carries a signature this key made (novox/hq ADR 0004).\n")
return nil
}
func brokerCommand(args []string) error {
if len(args) == 0 || args[0] != "show" {
return errors.New("broker show")
}
known, err := broker.FromEnvironment()
if errors.Is(err, broker.ErrNotConfigured) {
fmt.Printf("no broker configured. Set %s and %s.\n\n"+
"Until then tokens carry the signing key and the one-time secret, and say what they\n"+
"are missing. They cannot be used to join.\n",
broker.AddressVar, broker.CertificateVar)
return nil
}
if err != nil {
return err
}
fmt.Printf("address %s\n", known.Address)
fmt.Printf("fingerprint %s\n", known.Fingerprint)
fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" +
"node checks it before sending anything (novox/hq ADR 0004).\n")
return nil
}
// heardFrom says when a node was last heard from, in a form somebody can act on.
//
// "never" and "an hour ago" are different answers and are kept different. A node that has never
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
// picture of the mesh.
func heardFrom(n inventory.Node) string {
silent, ever := n.Silent()
switch {
case !ever:
return "never spoken"
case silent > SilentFor:
return "out of touch " + roughly(silent)
default:
return "here"
}
}
// roughly is a duration a person reads rather than parses.
func roughly(d time.Duration) string {
switch {
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%dh", int(d.Hours()))
default:
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
}
// showNode says what one machine reported about itself, in its own words.
//
// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what
// a machine can do is the one it gave — and its detail is half of that account. The mesh was
// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with
// nowhere to go: a person told a machine lacks something wants to know what the detector saw.
//
// It is also where "what should it be configured as" is read. The same line that gates an
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
node, err := inv.NodeByName(ctx, name)
if err != nil {
return err
}
fmt.Printf("%s\n", node.Name)
fmt.Printf(" last heard from %s\n", heardFrom(node))
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
// every internal node would be noise.
domain, err := inv.PublicDomainOf(ctx, name)
if err != nil {
return err
}
if domain != "" {
fmt.Printf(" public domain %s\n", domain)
}
held, err := inv.Profile(ctx, name)
if err != nil {
return err
}
if held == nil {
// Never reported is not the same as reported nothing, and the remedy differs: one is a
// machine that has not run the host yet, the other is a machine that ran it and can do
// nothing.
fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" +
" capability is refused here — run the host on it\n")
return nil
}
if len(held) == 0 {
fmt.Printf("\n it reported no capabilities at all\n")
return nil
}
fmt.Printf("\n what it can do, as it reported:\n")
for _, c := range held {
mark := "no "
if c.Present {
mark = "yes"
}
fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail)
}
assigned, err := inv.Assigned(ctx, name)
if err != nil {
return err
}
if len(assigned) > 0 {
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
}
return nil
}
cmd/mesh-controller/nodes_test.go
+99
View File
@@ -0,0 +1,99 @@
package main
import (
"strings"
"testing"
)
// **A read-shaped invocation is never a destructive write.**
//
// `node public-domain anchor` used to clear the domain. It reads like a question — it is what
// anybody types to find out what the answer is — and it silently took every routed name the node
// had. There is no output that makes up for that: by the time it prints, the fact is gone.
func TestAskingForANodesPublicDomainDoesNotTakeItAway(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
t.Fatal(err)
}
if err := publicDomain(ctx, open.inventory, []string{"anchor"}); err != nil {
t.Fatal(err)
}
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if domain != "example.test" {
t.Fatalf("asking what the domain is took it away: %q", domain)
}
}
// Clearing is a real thing to want — a machine that stops facing the outside composes no names —
// so it keeps a way to be said. What it stops being is what happens when nothing was said.
func TestClearingANodesPublicDomainIsAskedForByName(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
t.Fatal(err)
}
if err := publicDomain(ctx, open.inventory, []string{"anchor", "--clear"}); err != nil {
t.Fatal(err)
}
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if domain != "" {
t.Fatalf("--clear did not clear it: %q", domain)
}
}
// A domain sets it, as it always did.
func TestGivingANodeAPublicDomainSetsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := publicDomain(ctx, open.inventory, []string{"anchor", "example.test"}); err != nil {
t.Fatal(err)
}
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if domain != "example.test" {
t.Fatalf("got %q", domain)
}
}
// A domain and --clear say opposite things. Refused rather than one of them silently winning.
func TestADomainAndClearTogetherIsRefused(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
t.Fatal(err)
}
err := publicDomain(ctx, open.inventory, []string{"anchor", "other.test", "--clear"})
if err == nil {
t.Fatal("a domain and --clear together were accepted")
}
if !strings.Contains(err.Error(), "not both") {
t.Fatalf("the refusal does not say why: %v", err)
}
// And neither half happened.
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if domain != "example.test" {
t.Fatalf("a refused command changed something: %q", domain)
}
}
// Asking about a name the mesh has never heard of is a refusal, not "it has no public domain" —
// which is true of that name and says nothing.
func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
open := aMesh(t)
if err := publicDomain(t.Context(), open.inventory, []string{"nowhere"}); err == nil {
t.Fatal("a name the mesh does not know was answered as if it were a machine")
}
}
cmd/mesh-controller/plan.go
+873
View File
@@ -0,0 +1,873 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"net"
"strconv"
)
// working out what one machine should be.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// planFor works out everything a node should run, from what was assigned to it.
func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
inv := open.inventory
shelf, err := inv.Catalogue(ctx)
if err != nil {
return catalogue.Resolution{}, nil, err
}
assigned, err := inv.Assigned(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
capabilities, err := inv.ProfileOf(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
places, err := inv.Overlays(ctx)
if err != nil {
return catalogue.Resolution{}, nil, err
}
var site string
for _, p := range places {
if p.Name == nodeName {
site = p.Site
}
}
world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
world.Pinned, err = inv.PinsFor(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return catalogue.Resolution{}, nil, err
}
// What this mesh can answer with a record rather than a machine, and which record each of
// this node's modules was put on. Read across a context boundary by name, which is what
// crossing one is allowed to carry (novox/hq ADR 0008).
world.Licences, world.Using, err = licencesFor(ctx, open, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
// The domain this node composes its routed names under (novox/hq ADR 0066). A route
// contribution carries only a label; the resolver joins <label>.<public-domain> for this node,
// so the fact travels on the node it belongs to rather than being looked up where the name is
// composed.
publicDomain, err := inv.PublicDomainOf(ctx, nodeName)
if err != nil {
return catalogue.Resolution{}, nil, err
}
resolved, err := catalogue.Resolve(shelf, assigned,
catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities,
At: onNetwork[nodeName], PublicDomain: publicDomain}, world)
if err != nil {
return catalogue.Resolution{}, nil, err
}
// The credential for each thing this node takes from elsewhere. Made once and kept, so the
// password a provider is told to create is the one its consumer was given — and sealed to
// this node before it was ever written down, so nothing between here and there can read it.
for i, n := range resolved.Needs {
if n.ByRecord {
// Answered by something the mesh holds, so there is no pair-wise secret between two
// machines. Its key was supplied by a person and sealed to this node then; the mesh
// discarded the plaintext and cannot make another.
sealed, err := keyFor(ctx, open, n.From, nodeName, n.For)
if err != nil {
return catalogue.Resolution{}, nil, err
}
resolved.Needs[i].Sealed = sealed
// If this holder is the licence's manager, hand it the manager node's PUBLIC sealing key
// in its bound facts (novox/hq ADR 0050). It is safe to disclose — a public key — and it
// is what the manager module needs to re-seal a rotated refresh token to this same node,
// having been given no private key of its own. A consumer holder gets none.
pub, err := managerPublicKeyFor(ctx, open, inv, n.From, nodeName, n.For)
if err != nil {
return catalogue.Resolution{}, nil, err
}
if pub != "" {
serves := map[string]any{}
for k, v := range resolved.Needs[i].Serves {
serves[k] = v
}
serves["manager_public_key"] = pub
resolved.Needs[i].Serves = serves
// The manager holder: its empty pre-adoption refresh token is a waiting state, not a
// missing consumer key, so the declaration tolerates it rather than refusing.
resolved.Needs[i].Manager = true
}
continue
}
secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.For, n.From)
if err != nil {
// Said rather than skipped. A machine that resolves cleanly and receives no
// credential is one that will fail to authenticate at some later, less obvious
// moment.
return catalogue.Resolution{}, nil, fmt.Errorf(
"%s on %s needs %s from %s and no credential could be made for it: %w",
n.For, nodeName, n.Name, n.From, err)
}
resolved.Needs[i].Sealed = secret.ForConsumer
}
// Settings for everything that resolved, including modules nobody assigned directly: a
// requirement pulled in by something else is still configurable, and finding out that it is
// not only when you try would be an arbitrary line nobody could predict.
settings := catalogue.SettingsBy{}
var stray []string
for _, m := range resolved.Modules {
layers, err := inv.SettingsFor(ctx, nodeName, m.Module)
if err != nil {
return catalogue.Resolution{}, nil, err
}
if len(layers) == 0 {
continue
}
settings[m.Module] = layers
stray = append(stray, catalogue.UnusedSettings(m, layers)...)
}
if len(stray) > 0 {
// Somebody set something that reaches no file. Said here rather than discovered by the
// machine not behaving differently, which is the slowest way there is.
return catalogue.Resolution{}, nil, fmt.Errorf(
"these settings reach nothing:\n - %s", strings.Join(stray, "\n - "))
}
return resolved, settings, nil
}
// theRestOfTheMesh is what every other node holds and offers.
//
// Two things at once because they come from the same place — resolving the other nodes — and
// because both are facts about what is actually running rather than records that could disagree
// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node
// runs; neither is a table somebody keeps up to date.
//
// **Two passes over the others.** What a node offers the mesh needs that node resolved, and
// resolving it may need what the mesh offers. So the first pass takes brokered requirements on
// trust and answers only *what does each node offer*; the second answers everything with that in
// hand. Nothing is ever declared from the first.
func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
// Every node, not only the placed ones. A machine that was never put on the private network
// still runs modules, still holds claims, and still offers whatever it offers.
nodes, err := inv.Nodes(ctx)
if err != nil {
return catalogue.World{}, err
}
places, err := inv.Overlays(ctx)
if err != nil {
return catalogue.World{}, err
}
siteOf := map[string]string{}
for _, p := range places {
siteOf[p.Name] = p.Site
}
// Which machines are actually on the private network, and what they are called there. Not
// "has an address" — that was true of every placed machine and told you nothing about whether
// anything could reach it. It is what resolved the module.
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return catalogue.World{}, err
}
type candidate struct {
node catalogue.Node
assigned []string
}
var others []candidate
for _, n := range nodes {
if n.Name == exclude {
continue
}
theirs, err := inv.Assigned(ctx, n.Name)
if err != nil || len(theirs) == 0 {
continue
}
caps, _ := inv.ProfileOf(ctx, n.Name)
others = append(others, candidate{
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
At: onNetwork[n.Name]}, theirs})
}
offered := map[string][]catalogue.Provider{}
for _, o := range others {
got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
if err != nil {
// Their set does not resolve for some other reason. Not this node's problem to
// report, and nothing of theirs is running, so it offers nothing.
continue
}
for _, m := range got.Modules {
for _, name := range m.OffersAt(catalogue.ScopeMesh) {
// What that module says a consumer needs to know, with that node's settings on
// it: a port somebody moved on the provider is a port its consumers must be told
// about, and the two coming from different places is how they come to disagree.
assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
if err != nil {
return catalogue.World{}, err
}
serves := catalogue.ServedOn(m, name, assigned)
if len(serves) > 0 {
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
if err != nil {
return catalogue.World{}, err
}
serves, err = catalogue.Settle(serves, layers)
if err != nil {
return catalogue.World{}, err
}
}
offered[name] = append(offered[name], catalogue.Provider{
Node: o.node.Name, At: o.node.At, Serves: serves})
}
}
}
for k := range offered {
sort.Slice(offered[k], func(i, j int) bool {
return offered[k][i].Node < offered[k][j].Node
})
}
world := catalogue.World{Offered: offered}
for _, o := range others {
got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
if err != nil {
continue
}
world.Held = append(world.Held, got.Claims...)
}
return world, nil
}
// declarationFor is everything a node would be sent.
//
// One place, because there were three and one of them was written before credentials existed and
// silently produced a declaration missing them — a difference between what `plan` showed and what
// `plan --json` handed to anything reading it.
func declarationFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) {
gens, err := generators(ctx, open)
if err != nil {
return nil, err
}
// **Allocating, because `plan` is the send without the sending.** It is one machine, named by
// a person, who is asking what a push would do — so the port it shows and the secret it seals
// have to be the ones a push would use, and both are kept once chosen. Showing numbers that a
// later push would replace would make the command answer a question nobody asked.
//
// The line is not "a question may not write". It is who is asking and how often: this is a
// person, about one machine, on purpose. What may not write is the comparison the mesh runs
// over every machine to answer whether each is up to date — see Choosing.
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
}
// declarationWith is the same, for a caller that has already worked out the generators once and
// is about to use them for every node.
// Choosing says whether this composition may allocate what has not been allocated yet.
//
// **The comparison the mesh runs over every machine must not change what it is comparing.**
// Composing a declaration assigns each module a machine port and seals its secrets, and `status`
// composes one for every node to answer *is this machine running what I would send it* — so that
// question allocated, minted, wrote, and contended with the very machine it was asking about. A
// status polled every two seconds while a node applies is then two writers on the same rows,
// which is how it came to hang rather than answer.
//
// `plan` sits on the other side of this and allocates, because it is a person asking what a push
// would do to one named machine. The distinction is not question versus command; it is a person
// asking once about one machine versus the mesh asking continuously about all of them.
//
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
// machine "waiting" means — the read needs no number to be right about that.
type Choosing bool
const (
// Allocating is the send path: what is not assigned yet is assigned now and kept.
Allocating Choosing = true
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = false
)
func declarationWith(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
gens map[string]catalogue.Generator, choosing Choosing) ([]map[string]any, error) {
inv := open.inventory
grants, err := grantsFor(ctx, open, node)
if err != nil {
return nil, err
}
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
//
// **Assigned here rather than written by a module**, because a module is written once and
// assigned anywhere: any number it picks is a guess about a machine it has never seen. Made
// before the declaration is composed, because the container's mapping, the rule set and what a
// consumer is told are all derived from it.
// What this machine was already given, for a composition that may not allocate.
already := map[string]map[int]int{}
if choosing == Reading {
held, err := inv.PortsFor(ctx, node)
if err != nil {
return nil, err
}
for _, a := range held {
if already[a.Module] == nil {
already[a.Module] = map[int]int{}
}
already[a.Module][a.Wanted] = a.Machine
}
}
ports := map[string]map[int]int{}
for _, m := range plan.Modules {
for _, l := range m.Listens {
// **Only a port the module actually publishes is the mesh's to move.** A container's
// mapping is the thing that translates; without one the software binds what it binds,
// and an assignment would not move the service — it would open the wrong number in the
// rule set and leave the real one shut. Recorded either way, because this map means
// *where this module's port is on this machine* and every reader of it needs that
// answer whether or not the mesh was the one who chose it.
where, mayAssign := m.MachineSide(l.Port)
switch {
case mayAssign && choosing == Allocating:
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
if err != nil {
return nil, fmt.Errorf(
"%s needs %d reachable on %s and it could not be assigned: %w",
m.Module, l.Port, node, err)
}
where = at.Machine
case mayAssign:
// Whatever was chosen last time, and nothing if there was no last time.
if at, known := already[m.Module][l.Port]; known {
where = at
}
}
if ports[m.Module] == nil {
ports[m.Module] = map[int]int{}
}
ports[m.Module][l.Port] = where
}
}
// And each module's own secrets — a superuser password, an administrator, an account. Made
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
for _, m := range plan.Modules {
for name := range m.OwnSecrets {
// Minted on the send path and only read on every other. Making one is an insert, and
// a question that writes is a question that can block against the machine it is about.
var sealed string
var err error
if choosing == Allocating {
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
if err == nil && !held {
// Never issued, so this machine cannot be running it. Left out rather than
// invented: an empty string here would compose a declaration that differs
// from what would be sent, and the comparison this feeds would then be
// answering about a declaration nothing will ever push.
continue
}
}
if err != nil {
return nil, err
}
if needed[m.Module] == nil {
needed[m.Module] = map[string]string{}
}
needed[m.Module][name] = sealed
}
}
// And a certificate for this machine's name inside the mesh, when anything on it asks. Issued
// rather than stored: the node's key does not change, so signing again produces an equally
// valid certificate and there is nothing to keep in step.
var certificate, authority string
for _, m := range plan.Modules {
if m.Certificate == nil {
continue
}
issued, meshCA, err := certificateFor(ctx, open, node)
if err != nil {
return nil, err
}
certificate, authority = issued, meshCA
break
}
// And who else is on the private network, which is what a rule saying "from the mesh"
// resolves to. Every node's address, including this one's: a machine reaching itself by its
// own overlay address rather than by loopback is ordinary, and leaving it out would filter
// the node's own traffic to itself with no rule naming why.
private, err := onThePrivateNetwork(ctx, inv)
if err != nil {
return nil, err
}
// And every machine's name, so a container can reach one. The same set that writes the
// machine's own hosts file — one reading, so a container and its machine cannot disagree
// about where another machine is.
names, err := namesInTheMesh(ctx, inv)
if err != nil {
return nil, err
}
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
// `<node>.internal` names above, so a container — or an internal ACME validator — resolves a
// routed name to the proxy that serves it, mesh-wide. The mesh publishes the names it was told
// to serve and knows nothing about what they mean.
routes, err := routeNamesInTheMesh(ctx, open)
if err != nil {
return nil, err
}
for name, at := range routes {
names[name] = at
}
// The ports the mesh itself needs open, which no module declares. Read from the broker this
// control plane was told about rather than written down twice: the address a node is handed in
// its token and the port its machine must accept on are the same fact.
var foundation []int
if b, err := broker.FromEnvironment(); err == nil {
if _, port, err := net.SplitHostPort(b.Address); err == nil {
if n, err := strconv.Atoi(port); err == nil {
foundation = append(foundation, n)
}
}
}
return plan.Declaration(catalogue.Rendering{
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Foundation: foundation})
}
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
// ADR 0066).
//
// **Mesh-wide, so any container resolves any routed name to its proxy** — including an internal
// ACME validator, which cannot complete a challenge for a name it cannot reach. A routed name is
// composed on the consumer's node (from its label and that node's public domain) and served by the
// node answering the consumer's route requirement; this gathers both.
//
// It reads route names off resolutions rather than a table because there is no table: a route is a
// contribution, computed from what each node runs. Name-agnostic — a contribution counts as a
// routed name only because it carried a label the mesh composed, never because the mesh knows what
// "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
// the rest their names.
func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
inv := open.inventory
places, err := inv.Overlays(ctx)
if err != nil {
return nil, err
}
address := map[string]string{}
for _, p := range places {
if strings.TrimSpace(p.Address) != "" {
address[p.Name] = p.Address
}
}
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, open, n.Name)
if err != nil {
continue
}
for _, m := range plan.Modules {
for to := range m.Contributes {
values, asks, err := plan.ContributionsFrom(to, m.Module, settings)
if err != nil {
return nil, err
}
if !asks {
continue
}
// A routed name, and only that: a contribution the mesh composed a name for from a
// label it was given. A grant that happens to carry a `name` of its own — a database
// name — carries no label and is left alone.
if _, labelled := values["label"]; !labelled {
continue
}
name, _ := values["name"].(string)
if name == "" {
continue
}
// The node that serves it: whoever answers this consumer's route requirement, or
// this same node when the proxy is beside the consumer.
serving := n.Name
for _, need := range plan.Needs {
if need.Name == to && need.For == m.Module {
serving = need.From
break
}
}
if at := address[serving]; at != "" {
out[strings.ToLower(name)] = at
}
}
}
}
return out, nil
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
// the machine and whether it is on the private network, `identity` holds the authority and the
// key that machine reported. The process holding both grants asks each for its part
// (novox/hq ADR 0008).
func certificateFor(ctx context.Context, open *stores, node string) (string, string, error) {
inv := open.inventory
ident, err := open.Identity(ctx)
if err != nil {
return "", "", err
}
record, err := inv.NodeByName(ctx, node)
if err != nil {
return "", "", err
}
serving, err := ident.ServingKeyOf(ctx, record.ID)
if err != nil {
return "", "", err
}
if serving == "" {
// The machine joined before it had one, or never reported it. Said plainly, because the
// remedy is on the machine and no amount of pushing from here will produce one.
return "", "", fmt.Errorf(
"%s wants a certificate and has never told the mesh what key it serves with; it "+
"joins again to report one", node)
}
// The name it is certified for. Only a machine on the private network has one — a certificate
// for a name nothing resolves is a certificate nothing can check.
//
// With the catalogue, not without it. Being on the private network is a conclusion about what
// a node resolves to, so a nil shelf resolves nothing and every machine looks like it is on no
// network — which refused every certificate the mesh was asked for, and said the machine was
// not on a network it plainly was.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return "", "", err
}
where, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return "", "", err
}
name := where[node]
if name == "" {
return "", "", fmt.Errorf(
"%s wants a certificate and is not on the private network, so it has no name inside "+
"the mesh to be certified for", node)
}
issued, err := ident.Certify(ctx, node, name, serving)
if err != nil {
return "", "", err
}
authority, err := ident.EstablishAuthority(ctx)
if err != nil {
return "", "", err
}
return issued, authority.Certificate, nil
}
// grantsFor is every credential this node must create, because something elsewhere uses it.
//
// The mirror of what a consumer is given, and the half that makes the credential real: a password
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
// the mesh hands over something it cannot itself use.
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
inv := open.inventory
issued, err := inv.SecretsFrom(ctx, node)
if err != nil {
return nil, err
}
// Where each consumer is, so a provider that must reach back to one does not have to know how
// the mesh names machines.
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
if err != nil {
return nil, err
}
// What each consumer actually asked for, taken from that machine's own resolution rather than
// from a record beside it. A provider told to create a password and not what to create it for
// can do nothing with it, and the name a consumer wants is the consumer's to say.
out := make([]catalogue.Grant, 0, len(issued))
for _, s := range issued {
plan, settings, err := planFor(ctx, open, s.Consumer)
if err != nil {
// Their set does not resolve. Skipped rather than fatal: this node is not the place
// to report another machine's problem, and a grant for something that is not going to
// run would have the provider create a user nothing uses.
continue
}
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
if err != nil {
return nil, err
}
// A port in there is the consumer's software port until this. The consumer is on another
// machine, so the assignment that moved it is that machine's — fetched here rather than
// looked for in this one's, which is the whole reason the co-located fix could not reach
// this case (novox/hq 04-ISSUES/038, the cross-node half).
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
if err != nil {
return nil, err
}
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
from := s.ConsumerModule
if !asks {
// That module no longer wants this. Left empty, which is what the declaration reads
// as "nobody asks for it any more" — and is how a login is withdrawn rather than kept
// working for ever after its consumer went away.
from = ""
}
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
// remedy a short slug rather than a login a provider silently shortened.
slug := ""
for _, mm := range plan.Modules {
if mm.Module == s.ConsumerModule {
slug = mm.Slug
break
}
}
if from != "" {
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
return nil, err
}
}
out = append(out, catalogue.Grant{
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
From: from, Values: values, Slug: slug, Sealed: s.ForProvider})
}
return out, nil
}
func planCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("plan", flag.ContinueOnError)
// Because "one resource" does not tell you whether the settings landed. Being able to read
// the file before it is sent is the difference between believing a merge worked and knowing.
show := set.Bool("files", false, "print the files this node would be given")
// The declaration exactly as the node would receive it. For handing to something else --
// checking it against the host's own parser, most usefully, which is the only way to know
// that what the control plane emits is what the host accepts.
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("plan <node> [--files] [--json]")
}
args = positionals
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
plan, settings, err := planFor(ctx, open, args[0])
if err != nil {
return err
}
if len(plan.Modules) == 0 {
fmt.Printf("%s is assigned nothing\n", args[0])
return nil
}
if *asJSON {
resources, err := declarationFor(ctx, open, args[0], plan, settings)
if err != nil {
return err
}
body, err := json.MarshalIndent(
map[string]any{"declaration": 1, "resources": resources}, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
fmt.Printf("%s would run:\n", args[0])
for _, m := range plan.Modules {
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
}
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
// one module on the wrong machine, the others still run, and the remedy is to move this one.
for _, u := range plan.Unhostable {
for _, c := range u.Missing {
fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0]))
}
}
for _, c := range plan.Claims {
fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope)
}
// What this machine depends on that is not on it. Worth saying out loud: it is the only part
// of a node's set that stops working when a *different* machine goes away, and nothing else
// in this output would have told anybody that.
for _, n := range plan.Needs {
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
}
resources, err := declarationFor(ctx, open, args[0], plan, settings)
if err != nil {
return err
}
for module, layers := range settings {
for _, layer := range layers {
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
}
}
fmt.Printf("\n%d resource(s)\n", len(resources))
if *show {
for _, r := range resources {
content, ok := r["content"].(string)
if !ok {
continue
}
fmt.Printf("\n--- %v %v ---\n%s", r["id"], r["path"], content)
}
}
return nil
}
// licencesFor is what this node can be answered with by record, and what it was put on.
//
// A mesh with no licences at all is the ordinary case and must not be an error: every existing
// mesh is one, and a control plane that refused to plan because nobody had bought an API key
// would be unusable for the thing it already does.
func licencesFor(ctx context.Context, open *stores, node string) (
map[string][]catalogue.Record, map[string]map[string]catalogue.Record, error) {
held, err := open.Licences(ctx)
if err != nil {
return nil, nil, err
}
all, err := held.All(ctx)
if err != nil {
return nil, nil, err
}
if len(all) == 0 {
return nil, nil, nil
}
offered := map[string][]catalogue.Record{}
byName := map[string]catalogue.Record{}
for _, one := range all {
record := catalogue.Record{Name: one.Name, Serves: one.Serves}
offered[licences.Provision] = append(offered[licences.Provision], record)
byName[one.Name] = record
}
using := map[string]map[string]catalogue.Record{}
for _, one := range all {
holders, err := held.HoldersOf(ctx, one.Name)
if err != nil {
return nil, nil, err
}
for _, h := range holders {
if h.Node != node {
continue
}
if using[h.Module] == nil {
using[h.Module] = map[string]catalogue.Record{}
}
using[h.Module][licences.Provision] = byName[one.Name]
}
}
return offered, using, nil
}
// keyFor is the licence key sealed to one machine, for one module.
//
// **Empty is not an error here.** The mesh discarded the plaintext when it was supplied, so a
// holder recorded afterwards genuinely has no key — and the declaration refuses that by name,
// where the module and the path are both in view, rather than here.
func keyFor(ctx context.Context, open *stores, licence, node, module string) (string, error) {
held, err := open.Licences(ctx)
if err != nil {
return "", err
}
return held.KeyFor(ctx, licence, node, module)
}
// managerPublicKeyFor is the manager node's public sealing key, but only when (node, module) is the
// licence's manager holder — empty otherwise.
//
// It is delivered to the manager module in its bound facts so it can re-seal a rotated refresh token
// to this node (novox/hq ADR 0050). Public, so it travels in the clear like any other bound fact; and
// scoped to the manager holder alone, so a consumer never receives it and nothing invites a consumer
// to seal anything.
func managerPublicKeyFor(
ctx context.Context, open *stores, inv *inventory.Inventory, licence, node, module string,
) (string, error) {
held, err := open.Licences(ctx)
if err != nil {
return "", err
}
managerNode, managerModule, err := held.ManagerOf(ctx, licence)
if err != nil {
return "", err
}
if managerNode == "" || node != managerNode || module != managerModule {
return "", nil
}
return inv.SealingKeyOf(ctx, node)
}
// portsOn is one module's assignments on one machine, by the port the software uses.
func portsOn(
ctx context.Context, inv *inventory.Inventory, node, module string,
) (map[int]int, error) {
all, err := inv.PortsFor(ctx, node)
if err != nil {
return nil, err
}
out := map[int]int{}
for _, a := range all {
if a.Module == module {
out[a.Wanted] = a.Machine
}
}
return out, nil
}
cmd/mesh-controller/push.go
+536
View File
@@ -0,0 +1,536 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// reportUnhostable says which of a node's assigned modules the machine cannot run, once per push.
//
// A module whose declared capability has no detector on the machine is on the wrong machine. It is
// kept out of what the node is sent — the healthy modules beside it still converge — and named here
// so it is neither silently dropped nor a reason the whole node fails to push.
func reportUnhostable(node string, plan catalogue.Resolution) {
for _, u := range plan.Unhostable {
for _, c := range u.Missing {
fmt.Printf("%s not applied — %s\n", node, catalogue.WrongMachine(u.Module, c, node))
}
}
}
// sending it, and holding the link that carries it.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// serve is the control plane running: one connection to the broker, one queue, one consumer.
func serve(ctx context.Context) error {
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
// Established at start rather than on first use. A control plane that cannot sign is one
// whose declarations every node correctly refuses, and that should be a startup failure
// rather than something discovered at the first declaration.
key, err := ident.Establish(ctx)
if err != nil {
return err
}
fmt.Printf("signing as %s\n", key.Fingerprint()[:16])
management, err := broker.ManagementFromEnvironment()
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
return err
}
// Where the broker is and what to expect there, so a node can be told how to come back
// without a person and a new token.
known, err := broker.FromEnvironment()
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
return err
}
if errors.Is(err, broker.ErrNotConfigured) {
fmt.Printf("no broker address configured, so enrolled nodes will not be told how to "+
"reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar)
}
work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known}
server, err := link.Connect(work, work)
if err != nil {
return err
}
defer server.Close()
// And build results nobody was waiting for. A build triggered any other way than `build`
// would otherwise be reported into the void, which is the same as not reporting it.
server.Records(builds{inv})
// And what the catalogue decided a build meant. The builder's own result is already handled
// above; this is the other half — the control plane is the only one of the three that knows
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
if err := server.Follows(following{open}); err != nil {
return err
}
// And a catalogue that has just started, asking for what it missed. The same type answers
// both: what a build meant and what the builds were are two questions about one record.
if err := server.Answers(following{open}); err != nil {
return err
}
return server.Serve(ctx)
}
// declare sends one node a declaration, signed.
//
// Signed here rather than trusted from the broker: a node connects to the broker and takes
// instruction from the control plane behind it, and those are two identities. If a node believed
// whatever arrived on its queue, a compromised broker could forge declarations — and since the
// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004).
func declare(ctx context.Context, args []string) error {
if len(args) != 2 {
return errors.New("declare <node> <declaration.json>")
}
node, path := args[0], args[1]
raw, err := os.ReadFile(path)
if err != nil {
return err
}
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
// The node has to exist before it can be told anything. Publishing to a queue nobody consumes
// would sit there looking like success.
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
return nil
}
// OverlayCIDRVar is the range the mesh allocates node addresses from.
const OverlayCIDRVar = "MESH_OVERLAY_CIDR"
// pushCommand sends nodes everything they should be: their place on the network, and what their
// assignments resolve to.
//
// One declaration, not two. A node holding its network and not its modules, or the reverse, is
// half-configured for as long as that lasts — and the two are computed from the same picture of
// the mesh, so sending them apart would let them disagree.
func pushCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("push", flag.ContinueOnError)
// Only the machines that need it.
//
// **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is
// a scheduler over this, and building the scheduler first would mean two paths to one act
// with nothing to compare them against. A person can run this; so can cron; so can whatever
// eventually watches.
behind := set.Bool("behind", false,
"only machines whose last declaration was refused or partly failed")
// For a named node, wait until it reports applying exactly what it was sent, so `push <node>`
// means "this node is now what it was told" — a command right after does not race the apply
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
wait := set.Duration("wait", 0,
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
args = positionals
if len(args) > 1 {
return errors.New("push [<node>] [--behind] — one node, or all of them")
}
if len(args) == 1 && *behind {
// Naming a machine and asking for the ones that need it are two different requests, and
// guessing which was meant would sometimes push to a machine somebody did not name.
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
"other asks which machines need one")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
// Every node, not only the ones on the private network. A machine that was never given the
// network module still takes modules, and iterating the network here is what used to make
// "on the network" and "managed" the same thing.
nodes, err := inv.Nodes(ctx)
if err != nil {
return err
}
// Which machines are not in the state they were sent, when that is what was asked for.
var needsOne map[string]inventory.Doing
if *behind {
wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
if err != nil {
return err
}
needsOne = map[string]inventory.Doing{}
for _, d := range wrong {
needsOne[d.Node] = d
}
// **And every machine not running what the mesh would send it.** "Behind" used to mean
// only "failed or refused", so a machine that applied cleanly and whose declaration has
// since changed was not behind — and novox/hq ADR 0010's question, *did my change go
// out?*, was answerable only for the machines that broke.
would, err := wouldSend(ctx, open, nodes)
if err != nil {
return err
}
waiting, err := inv.Waiting(ctx, would)
if err != nil {
return err
}
for _, m := range waiting {
if _, already := needsOne[m.Node]; already {
continue
}
needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"}
}
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
fmt.Println("every machine is doing what it was told")
return nil
}
}
gens, err := generators(ctx, open)
if err != nil {
return err
}
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
// Which machines this push is about, before any of them is worked out.
var asked []string
for _, n := range nodes {
if len(args) == 1 && n.Name != args[0] {
continue
}
if *behind {
doing, needs := needsOne[n.Name]
if !needs {
continue
}
// A machine that has been failing the same way for a long time is not going to stop
// because it was asked again. Said, and pushed to anyway — refusing would leave no
// way to retry after fixing the cause, and this is a command somebody ran.
//
// Only for machines that reported something. One that is merely waiting has no report
// to be old, and saying it had been failing since the zero time would be a sentence
// about nothing.
if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour {
fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+
"unlikely to be timing\n",
n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04"))
}
}
asked = append(asked, n.Name)
}
sending, refusals := composeEach(asked, func(node string) ([]map[string]any, error) {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
return nil, err
}
// A module assigned here that this machine cannot host is said and left out, not fatal: the
// healthy modules beside it are still resolved and sent. Reported so it is not silently
// dropped — the remedy is to move it, and until then the rest of the node converges.
reportUnhostable(node, plan)
// The private network is in here with everything else. It used to be composed separately
// and prepended, which meant every machine with an address was on it and no machine could
// be kept off. It is a module now, so it arrives the way a module does.
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
})
sentDigest := map[string]string{}
for _, s := range sending {
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
if err != nil {
return err
}
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
// After it is away, not before. A digest recorded for something that failed to send would
// make the machine look current for a declaration it never received.
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
digest := digestOf(body)
if err := inv.RecordSent(ctx, record.ID, digest); err != nil {
return err
}
sentDigest[s.node] = digest
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
}
fmt.Printf("\n%d node(s) told\n", len(sending))
// A named node is a request to make THAT node current now, so it waits for the node to say it
// applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking
// on the slowest machine would hold back the report on all the others.
if *wait > 0 && len(args) == 1 {
if err := waitForApplied(ctx, inv, args[0], sentDigest[args[0]], *wait); err != nil {
return err
}
}
return couldNotBeResolved(refusals, len(sending))
}
// waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the
// wait runs out. A report of failure or refusal for that same declaration ends the wait at once —
// there is nothing to wait for, and the reason is the node's own.
func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest string, wait time.Duration) error {
if digest == "" {
return nil // nothing was sent to this node
}
deadline := time.Now().Add(wait)
for {
doing, said, err := inv.DoingOf(ctx, node)
if err != nil {
return err
}
if said && doing.Declared == digest {
switch doing.Outcome {
case inventory.OutcomeApplied:
fmt.Printf("%s applied it\n", node)
return nil
case inventory.OutcomeFailed:
return fmt.Errorf("%s applied what it was sent but %d resource(s) failed", node, len(doing.Failed))
case inventory.OutcomeRefused:
return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused)
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not report applying what it was sent within %s "+
"(it may still be converging; check `status`)", node, wait)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
}
// readyNode is one machine and the declaration it would be sent.
type readyNode struct {
node string
resources []map[string]any
}
// composeEach works out what each named machine should be, and never lets one machine's answer
// decide another's.
//
// **A machine whose set cannot be worked out is that machine's problem** (novox/hq ADR 0066). A
// whole-mesh push used to refuse outright when any one node failed to resolve, so a single
// unanswerable requirement on a single machine — one module requiring a provision nobody had
// assigned a provider for — left every other machine in the mesh unconverged, including machines
// with no relation to it at all. Nothing was sent anywhere, and the machines that could not be sent
// were the ones with nothing wrong with them.
//
// It is the same rule a92c11b established one level down, where an un-hostable module stopped
// taking down the healthy modules beside it, applied one level up: **the blast radius of a fault is
// the thing that has it.** What could not be worked out is named and returned, so a push still ends
// with a non-zero outcome and nobody mistakes a partial convergence for a whole one.
//
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
// across two machines that must agree — and dropped here, where it never did.
func composeEach(names []string,
compose func(node string) ([]map[string]any, error)) ([]readyNode, []string) {
var sending []readyNode
var refusals []string
for _, name := range names {
resources, err := compose(name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
if len(resources) == 0 {
fmt.Printf("%s is assigned nothing — skipped\n", name)
continue
}
sending = append(sending, readyNode{name, resources})
}
return sending, refusals
}
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
//
// **After the rest have been sent, never instead of sending them.** It is still an error, because
// the mesh is not in the state somebody asked for and a command that exits cleanly having skipped a
// machine is a command that lies. What it must not do is decide anything about the machines beside
// it, which is why it says how many were sent.
func couldNotBeResolved(refusals []string, sent int) error {
if len(refusals) == 0 {
return nil
}
return fmt.Errorf(
"%d node(s) could not be resolved and were not sent. %d other node(s) were:\n\n%s",
len(refusals), sent, strings.Join(refusals, "\n\n"))
}
// sendTo resolves and sends to exactly the machines named, or refuses without sending anything.
//
// The all-or-nothing rule push deliberately does NOT follow, and for a reason that holds here and
// not there: a rotation that reached the
// consumer and refused on the provider would leave one end holding a credential the other has
// never heard of — which is the state this whole mechanism exists to make impossible.
func sendTo(ctx context.Context, open *stores, names []string) error {
inv := open.inventory
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
gens, err := generators(ctx, open)
if err != nil {
return err
}
type ready struct {
node string
resources []map[string]any
}
var sending []ready
var refusals []string
for _, name := range names {
plan, settings, err := planFor(ctx, open, name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
reportUnhostable(name, plan)
resources, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
sending = append(sending, ready{name, resources})
}
if len(refusals) > 0 {
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
len(refusals), strings.Join(refusals, "\n\n"))
}
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
for _, s := range sending {
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
if err != nil {
return err
}
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
return err
}
record, err := inv.NodeByName(ctx, s.node)
if err != nil {
return err
}
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
return err
}
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources))
}
return nil
}
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
//
// Over the same bytes that are sent, so the comparison is of the thing itself rather than of
// something derived beside it that could drift from it.
func digestOf(body []byte) string {
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:])
}
// wouldSend is the digest of what each machine should be right now.
//
// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
func wouldSend(ctx context.Context, open *stores,
nodes []inventory.Node) (map[string]string, error) {
gens, err := generators(ctx, open)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, open, n.Name)
if err != nil {
continue
}
resources, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
if err != nil {
continue
}
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources})
if err != nil {
return nil, err
}
out[n.Name] = digestOf(body)
}
return out, nil
}
cmd/mesh-controller/push_test.go
+74
View File
@@ -0,0 +1,74 @@
package main
import (
"errors"
"strings"
"testing"
)
// One machine that cannot be worked out is not a reason to leave the mesh unconverged.
//
// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on
// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine
// from being sent anything — machines with no relation to the fault, and nothing wrong with them.
// The failure and the punishment were on different machines.
//
// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on
// the wrong machine no longer refuses the whole node), applied one level up.
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
sending, refusals := composeEach(
[]string{"anchor", "home-server", "laptop"},
func(node string) ([]map[string]any, error) {
if node == "anchor" {
return nil, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
}
return []map[string]any{{"id": node + ".thing"}}, nil
})
var told []string
for _, s := range sending {
told = append(told, s.node)
}
if strings.Join(told, ",") != "home-server,laptop" {
t.Errorf("a machine with nothing wrong with it was not sent: %v", told)
}
if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") ||
!strings.Contains(refusals[0], "acme-ca") {
t.Errorf("the machine that could not be worked out was not named with its reason: %v",
refusals)
}
}
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
sending, refusals := composeEach([]string{"spare"},
func(string) ([]map[string]any, error) { return nil, nil })
if len(sending) != 0 || len(refusals) != 0 {
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
}
}
// A push that skipped a machine still ends badly, and says what was sent.
//
// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command
// exits non-zero — but it says how many machines it did reach, because the old message ("nothing
// was sent") was the very claim that had become untrue.
func TestASkippedMachineIsStillAnError(t *testing.T) {
if err := couldNotBeResolved(nil, 3); err != nil {
t.Fatalf("a push that resolved every machine reported a problem: %v", err)
}
err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2)
if err == nil {
t.Fatal("a push that could not work out a machine reported success")
}
said := err.Error()
if strings.Contains(said, "nothing was sent") {
t.Errorf("the push says nothing was sent, and it sent two machines: %q", said)
}
for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} {
if !strings.Contains(said, want) {
t.Errorf("the refusal does not say %q: %q", want, said)
}
}
}
cmd/mesh-controller/readable.go
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"encoding/json"
"fmt"
"sort"
"time"
)
// The same answers, in a shape something other than a person can read.
//
// A board reads through interfaces and holds nothing (novox/hq 03-DESIGN/01-to-be/11-a-board.md).
// Everything it needs is already answered by these commands — as text, for people, which is not
// something a page can read. So each of them can say it again as JSON.
//
// **`--json` rather than a serving API**, because nothing needs one yet: whatever serves a board
// runs the command, and the constraint in the design holds either way — the board never touches a
// context's store. An API is the larger thing and should wait until something is asking for it.
//
// **These shapes are hard to change once anything is built against them.** So they stay close to
// what the domain already calls things, and carry no summary field that would have to be kept
// true. Nothing here is derived that a reader could not derive.
// meshStatus is what `status --json` says: the three questions, in the order they are asked.
type meshStatus struct {
// Wrong is every machine whose last declaration was refused or partly failed.
Wrong []machineDoing `json:"wrong"`
// Quiet is every machine not heard from lately. Not the same as wrong: new, switched off and
// unreachable are not "tried and could not".
Quiet []machineQuiet `json:"quiet"`
// Behind is every module built from something older than its source has.
Behind []moduleBehind `json:"behind"`
// Waiting is every machine not running what the mesh would send it. The same question as
// Behind one level down: that says the catalogue is old, this says a machine is — and only
// this one has somebody's change waiting inside it.
Waiting []machineWaiting `json:"waiting"`
// Reported is every machine's last word beside when it was last sent a declaration. A
// machine whose report is newer than its send has acted on the current declaration; one
// whose is older is still working — and Waiting cannot tell those apart, because the sent
// digest is recorded at send, not at apply.
Reported []machineReported `json:"reported"`
// Unresolved is every machine that cannot be worked out at all, with what the mesh said when
// it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
// there is nothing to compare it against and nothing it can be behind — which is why a
// document without this field described a wholly blocked mesh as a well one.
//
// Per machine, and data. One node failing must never take the document away from a reader
// asking about the others.
Unresolved []machineUnresolved `json:"unresolved"`
// Network is why the private network could not be computed, when it could not; absent when it
// could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
// network, and a mesh whose hub is that node has no hub.
Network string `json:"network,omitempty"`
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
// "none at all".
Machines int `json:"machines"`
}
type machineUnresolved struct {
Node string `json:"node"`
// Problem is the mesh's own words, whole — newlines and all. It lists every requirement that
// could not be met, and a first line alone would name one of them and hide the rest.
Problem string `json:"problem"`
}
type machineDoing struct {
Node string `json:"node"`
// Outcome is refused or failed. Kept distinct all the way out: they are fixed in different
// places, and one word for both sends half the readers to the wrong one.
Outcome string `json:"outcome"`
Refused string `json:"refused,omitempty"`
Failed []struct {
ID string `json:"id"`
Error string `json:"error"`
} `json:"failed,omitempty"`
Applied int `json:"applied"`
At time.Time `json:"at"`
}
type machineReported struct {
Node string `json:"node"`
Outcome string `json:"outcome"`
At *time.Time `json:"at,omitempty"`
Sent *time.Time `json:"sent,omitempty"`
// Current is whether the last report names the declaration last sent. Not derivable from
// the timestamps beside it: an apply begun under the previous declaration reports after the
// next send, newer and still about the old words.
Current bool `json:"current"`
}
type machineWaiting struct {
Node string `json:"node"`
// Never is true when nothing has ever been sent to it. Not out of date: nobody has ever asked
// this machine to be anything, and the two read differently to whoever is looking.
Never bool `json:"never"`
Sent *time.Time `json:"sent,omitempty"`
}
type machineQuiet struct {
Node string `json:"node"`
// LastSeen is absent when the machine has never spoken, which is a different thing from
// having been quiet for a while.
LastSeen *time.Time `json:"lastSeen,omitempty"`
}
type moduleBehind struct {
Module string `json:"module"`
BuiltFrom string `json:"builtFrom"`
Head string `json:"head"`
On []string `json:"on"`
}
// statusAsJSON answers the same questions as the text form, from the same reading.
//
// **It takes the whole reading rather than a growing argument list**, which is what let a new
// answer be added to the text form and forgotten here — the two are one function's output in two
// shapes, and they must not be able to differ about what was asked.
//
// It never fails on account of the mesh. Every per-machine problem in here is a field, so one
// machine that cannot be worked out cannot stop a caller reading about the others: a
// machine-readable interface that stops being machine-readable exactly when something is wrong is
// one nobody can build an alarm on.
func statusAsJSON(asked answers) ([]byte, error) {
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
behind, sources := asked.behind, asked.sources
waiting, reported := asked.waiting, asked.reported
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
Network: asked.network}
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
}
sort.Slice(out.Unresolved, func(i, j int) bool {
return out.Unresolved[i].Node < out.Unresolved[j].Node
})
for _, r := range reported {
out.Reported = append(out.Reported, machineReported{
Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
}
for _, m := range waiting {
out.Waiting = append(out.Waiting, machineWaiting{
Node: m.Node, Never: m.Never, Sent: m.SentAt})
}
for _, d := range wrong {
row := machineDoing{
Node: d.Node, Outcome: d.Outcome, Refused: d.Refused, Applied: d.Applied, At: d.At,
}
for _, f := range d.Failed {
row.Failed = append(row.Failed, struct {
ID string `json:"id"`
Error string `json:"error"`
}{ID: f.ID, Error: f.Error})
}
out.Wrong = append(out.Wrong, row)
}
for _, n := range quiet {
row := machineQuiet{Node: n.Name}
if !n.LastSeen.IsZero() {
seen := n.LastSeen
row.LastSeen = &seen
}
out.Quiet = append(out.Quiet, row)
}
for module, on := range behind {
from := sources[module]
out.Behind = append(out.Behind, moduleBehind{
Module: module, BuiltFrom: from.BuiltFrom, Head: from.Head, On: on,
})
}
return json.MarshalIndent(out, "", " ")
}
// say prints a value as JSON, for the commands that can answer either way.
func say(value any) error {
body, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
cmd/mesh-controller/readable_test.go
+141
View File
@@ -0,0 +1,141 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// The shape something other than a person reads.
//
// Hard to change once anything is built against it, so it stays close to what the domain already
// calls things and carries no summary field that would have to be kept true.
func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
behind map[string][]string, sources map[string]inventory.Source) map[string]any {
t.Helper()
body, err := statusAsJSON(answers{
wrong: wrong, nodes: nodes, quiet: quiet, behind: behind, sources: sources})
if err != nil {
t.Fatal(err)
}
var parsed map[string]any
if err := json.Unmarshal(body, &parsed); err != nil {
t.Fatalf("what a board would read is not JSON: %v", err)
}
return parsed
}
func TestRefusedAndFailedStayDistinctAllTheWayOut(t *testing.T) {
// They are fixed in different places, so one word for both would send half the readers of a
// page to the wrong one.
got := statusOf(t,
[]inventory.Doing{
{Node: "one", Outcome: inventory.OutcomeRefused, Refused: "a file needs a path"},
{Node: "two", Outcome: inventory.OutcomeFailed, Applied: 3,
Failed: []inventory.FailedResource{{ID: "shell.pkg", Error: "target not found"}}},
},
[]inventory.Node{{Name: "one"}, {Name: "two"}}, nil, nil, nil)
wrong, _ := got["wrong"].([]any)
if len(wrong) != 2 {
t.Fatalf("got %v", got["wrong"])
}
first, _ := wrong[0].(map[string]any)
if first["outcome"] != inventory.OutcomeRefused || first["refused"] == nil {
t.Fatalf("a refusal did not survive: %v", first)
}
second, _ := wrong[1].(map[string]any)
if second["outcome"] != inventory.OutcomeFailed {
t.Fatalf("a failure did not survive: %v", second)
}
if second["applied"] != float64(3) {
// "Three of eight" and "none of eight" are different machines.
t.Fatalf("what did apply was not carried: %v", second)
}
}
func TestAMachineThatNeverSpokeSaysSoByOmission(t *testing.T) {
// Never heard from and quiet for a while are different situations, and a zero time would read
// as a date in 1970 on any page that formatted it.
got := statusOf(t, nil,
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
nil, nil)
quiet, _ := got["quiet"].([]any)
if len(quiet) != 2 {
t.Fatalf("got %v", got["quiet"])
}
never, _ := quiet[0].(map[string]any)
if _, said := never["lastSeen"]; said {
t.Fatalf("a machine that never spoke carries a time: %v", never)
}
away, _ := quiet[1].(map[string]any)
if _, said := away["lastSeen"]; !said {
t.Fatalf("a machine that has been quiet carries no time: %v", away)
}
}
func TestNothingWrongIsAnEmptyListRatherThanNothing(t *testing.T) {
// A page distinguishing "no machines are wrong" from "this field is missing" would have to
// handle both, and null is the one that gets forgotten.
got := statusOf(t, nil, []inventory.Node{{Name: "a", LastSeen: time.Now()}}, nil, nil, nil)
for _, key := range []string{"wrong", "quiet", "behind"} {
list, ok := got[key].([]any)
if !ok {
t.Fatalf("%q is %T, not a list", key, got[key])
}
if len(list) != 0 {
t.Fatalf("%q is not empty: %v", key, list)
}
}
// And how many machines there are, so a reader can tell "none wrong" from "none at all".
if got["machines"] != float64(1) {
t.Fatalf("got %v", got["machines"])
}
}
func TestWhatIsBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
// A module being out of date is a fact about the catalogue; machines running the old one is
// the thing with consequences.
got := statusOf(t, nil, nil, nil,
map[string][]string{"shell": {"workstation", "laptop"}},
map[string]inventory.Source{"shell": {BuiltFrom: "aaaaaaa1", Head: "bbbbbbb2"}})
behind, _ := got["behind"].([]any)
if len(behind) != 1 {
t.Fatalf("got %v", got["behind"])
}
row, _ := behind[0].(map[string]any)
if row["module"] != "shell" || row["builtFrom"] != "aaaaaaa1" || row["head"] != "bbbbbbb2" {
t.Fatalf("got %v", row)
}
on, _ := row["on"].([]any)
if len(on) != 2 {
t.Fatalf("the machines running the old one are not named: %v", row)
}
}
func TestNoSecretIsInWhatABoardReads(t *testing.T) {
// Everything here comes from the mesh's own records, which hold no readable secret — but a
// shape a page is built against is exactly where one would eventually be added for
// convenience, so this says it out loud.
body, err := statusAsJSON(answers{
wrong: []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
Refused: "resource \"x\": a file needs a path"}},
nodes: []inventory.Node{{Name: "a"}},
refused: map[string]string{"a": "nothing provides \"database\", wanted by web"},
})
if err != nil {
t.Fatal(err)
}
for _, word := range []string{"password", "secret", "sealed", "credential", "token"} {
if strings.Contains(strings.ToLower(string(body)), word) {
t.Fatalf("what a board reads carries a %q field:\n%s", word, body)
}
}
}
cmd/mesh-controller/rotate.go
+117
View File
@@ -0,0 +1,117 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"sort"
)
// rotateCommand replaces a credential and moves both ends together.
//
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
// password on every adoption and updated only the provider's row. Consumers on three nodes held
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
//
// Three things make that impossible here, and all three are deliberate:
//
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
// consumer's password touches one role and leaves every other consumer alone — and the list of who
// is affected is a query rather than an assumption.
//
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
// and left the sending to whoever remembered is the fault above, exactly.
//
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
// has half-rotated.
func rotateCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("rotate", flag.ContinueOnError)
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
// and rotating the other nine would be a great deal of disruption for one suspicion.
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("rotate <provision> [--consumer <machine>]")
}
provision := positionals[0]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
holders, err := inv.HoldersOf(ctx, provision, *only)
if err != nil {
return err
}
if len(holders) == 0 {
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
// and a rotation somebody believes happened is worse than one they know did not.
if *only != "" {
return fmt.Errorf(
"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
"what it does hold", *only, provision, *only)
}
return fmt.Errorf(
"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
provision)
}
// Every machine at both ends, named before anything changes. A person about to rotate a
// production credential is entitled to know the blast radius before it is the past tense.
affected := map[string]bool{}
for _, h := range holders {
affected[h.Consumer] = true
affected[h.Provider] = true
}
machines := make([]string, 0, len(affected))
for name := range affected {
machines = append(machines, name)
}
sort.Strings(machines)
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
for _, h := range holders {
// The module, because a machine may hold several credentials for one provision and
// rotating "anchor's database password" now means rotating three of them.
fmt.Printf(" %s on %s, from %s\n", h.ConsumerModule, h.Consumer, h.Provider)
}
for _, h := range holders {
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider); err != nil {
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
// the remedy is to run this again rather than to repair anything — but a machine
// whose secret was discarded and not resent is holding a credential the provider is
// about to stop honouring, and that is worth knowing now.
return fmt.Errorf(
"rotating %s for %s on %s from %s: %w\n\nSome credentials were discarded and "+
"not yet sent. Run this again once the cause is fixed",
h.Provision, h.ConsumerModule, h.Consumer, h.Provider, err)
}
}
// **Both ends, in one send.** There is a window either way — a role's password changes on the
// provider and the file changes on the consumer, and they cannot be simultaneous — so the
// honest thing is to make it as short as the broker allows and to never leave it open across
// a command boundary, where it depends on somebody's memory.
fmt.Printf("\nsending to both ends:\n")
if err := sendTo(ctx, open, machines); err != nil {
return fmt.Errorf(
"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
"Nothing on those machines has changed yet, so what is running keeps working "+
"until the provider next applies. Fix the cause and run `push --behind`", err)
}
fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
"changed cannot authenticate — `status` says who is still behind\n", len(machines))
return nil
}
cmd/mesh-controller/secret.go
+136
View File
@@ -0,0 +1,136 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"strings"
)
// secretCommand gives the mesh a value it must carry and could not have invented.
//
// **Every other secret in this mesh is one the mesh made** — generated, sealed to the machine that
// will use it, and never readable again. That is right for something coming into existence, and
// wrong for something that already exists: a database created last year has the password it was
// created with, and generating a new one puts 32 random bytes where a working credential was.
// The machine applies it, reports success, and whatever reads it fails to authenticate somewhere
// else entirely — with the mesh insisting the secret was delivered, which it was.
//
// So this is the entry point for **adopting** something already running. The store has carried
// the distinction since the beginning: a module secret records whether it was `made` or
// `accepted`, and refuses to invent a replacement for the second. Nothing until now could write
// one, so the only accepted secret in the mesh was the broker account issued to a build machine.
//
// The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
// **The only difference between the two is where the value came from.**
func secretCommand(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "accept" {
return errors.New("secret accept <node> <module> <name> [--from <file>]")
}
rest, flags := split(args[1:])
set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
from := set.String("from", "",
"read the value from this file instead of asking (use - for standard input)")
if err := set.Parse(flags); err != nil {
return err
}
if len(rest) != 3 {
return errors.New("secret accept <node> <module> <name> [--from <file>]")
}
node, module, name := rest[0], rest[1], rest[2]
value, err := valueFor(node, module, name, *from)
if err != nil {
return err
}
value = asSupplied(value)
if value == "" {
return errors.New("there is nothing to seal")
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
return err
}
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
// machine and the mesh cannot read it again.
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
fmt.Printf(" run `push %s` to send it\n", node)
return nil
}
// split separates what this command is about from how it was asked.
//
// **Because the standard library stops parsing at the first non-flag argument.** With the
// positionals first — which is the order that reads correctly — everything after them is left
// sitting in the arguments, so `secret accept a b c --from -` arrives as five positionals and the
// flag is never seen. The host's own parser carries the same note, and the fault it names is
// worse than this one: there, a flag somebody passed was silently ignored and the command
// succeeded anyway.
func split(args []string) (positional, flags []string) {
for i, arg := range args {
if strings.HasPrefix(arg, "-") {
return args[:i], args[i:]
}
}
return args, nil
}
// asSupplied is the value with its line ending removed and nothing else.
//
// **A file has a trailing newline and a password does not**, so the ending goes — a credential
// wrong by one byte fails in a way nobody connects to how it was supplied.
//
// **And only the ending.** Trimming both ends is the obvious thing and it is wrong: a password
// chosen with a leading space is one the mesh would then deliver as a different password, silently,
// with the operator certain they had supplied it correctly.
func asSupplied(raw string) string {
return strings.TrimRight(raw, "\r\n")
}
// valueFor gets the secret without putting it somewhere it can be read afterwards.
//
// **Not an argument, and there is no flag that takes one.** A value on the command line is in the
// shell's history, in the process list for as long as it runs, and in whatever collects either.
// The paths here are a file the operator already has, or a prompt that does not echo — the same
// two ways a model-access key is supplied (novox/hq ADR 0024).
func valueFor(node, module, name, from string) (string, error) {
switch {
case from == "-":
body, err := io.ReadAll(os.Stdin)
if err != nil {
return "", err
}
return string(body), nil
case from != "":
body, err := os.ReadFile(from)
if err != nil {
return "", err
}
return string(body), nil
default:
// The same path a model-access key takes, and for the same reason: a value given as an
// argument is in the shell's history and in the process list. Read from standard input,
// echoed nowhere by this program.
fmt.Fprintf(os.Stderr,
"reading %s's %q for %s from standard input; it is not echoed anywhere\n",
module, name, node)
line, err := bufio.NewReader(os.Stdin).ReadString('\n')
if err != nil && line == "" {
return "", fmt.Errorf("nothing was given on standard input: %w", err)
}
return line, nil
}
}
cmd/mesh-controller/secret_test.go
+111
View File
@@ -0,0 +1,111 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// A file has a trailing newline and a password does not.
//
// The failure this prevents is the worst kind to diagnose: the credential is delivered, the
// machine applies it, everything reports success, and authentication fails one byte from correct
// somewhere else entirely.
func TestATrailingNewlineIsNotPartOfTheSecret(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "password")
if err := os.WriteFile(path, []byte("the-database-password\n"), 0o600); err != nil {
t.Fatal(err)
}
got, err := valueFor("anchor", "umami", "database", path)
if err != nil {
t.Fatal(err)
}
if asSupplied(got) != "the-database-password" {
t.Fatalf("read %q", got)
}
}
// A password may contain spaces, and they are the operator's.
//
// Trimming both ends is the obvious thing and it is wrong: a value chosen with a leading space is
// a value the mesh would silently deliver as a different one.
func TestOnlyLineEndingsAreRemoved(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "password")
if err := os.WriteFile(path, []byte(" spaces matter \n"), 0o600); err != nil {
t.Fatal(err)
}
got, err := valueFor("anchor", "umami", "database", path)
if err != nil {
t.Fatal(err)
}
if asSupplied(got) != " spaces matter " {
t.Fatalf("the value was altered beyond its line ending: %q", got)
}
}
// A file that is not there is said plainly, rather than becoming an empty secret.
func TestAMissingFileIsRefused(t *testing.T) {
if _, err := valueFor("anchor", "umami", "database",
filepath.Join(t.TempDir(), "absent")); err == nil {
t.Fatal("a missing file produced a value")
}
}
// The command refuses what it cannot act on, rather than acting on part of it.
func TestTheArgumentsAreRequired(t *testing.T) {
for _, args := range [][]string{
{},
{"accept"},
{"accept", "anchor"},
{"accept", "anchor", "umami"},
{"give", "anchor", "umami", "database"},
} {
if err := secretCommand(t.Context(), args); err == nil {
t.Errorf("%v was accepted", args)
}
}
}
// The invocation that actually gets used, which the first version of these tests never tried.
//
// Every case here was a rejection, so the command was broken in the one way that matters — it
// refused what it is for — and the tests were green. The lab found it at the first call.
func TestTheArgumentsAndTheFlagAreBothSeen(t *testing.T) {
rest, flags := split([]string{"anchor", "umami", "database", "--from", "-"})
assert(t, len(rest) == 3, "the three positionals were not kept: %v", rest)
assert(t, len(flags) == 2, "the flag was not separated: %v", flags)
// And with no flag at all, which is the interactive form.
rest, flags = split([]string{"anchor", "umami", "database"})
assert(t, len(rest) == 3 && len(flags) == 0, "%v / %v", rest, flags)
// A flag before the positionals still works, because somebody will write it that way.
rest, flags = split([]string{"--from", "/tmp/x"})
assert(t, len(rest) == 0 && len(flags) == 2, "%v / %v", rest, flags)
}
// And the wiring, not just the helper.
//
// Testing `split` alone left the command able to ignore it entirely — removing the call changed
// no test. This reaches secretCommand: with a `--from` naming a file that is not there, the
// complaint must be about the file. A complaint about usage would mean the flag was never seen.
func TestTheCommandItselfSeesTheFlag(t *testing.T) {
err := secretCommand(t.Context(),
[]string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"})
if err == nil {
t.Fatal("a missing file was accepted")
}
if strings.Contains(err.Error(), "secret accept <node>") {
t.Fatalf("the command did not see its flag and complained about usage instead: %v", err)
}
}
func assert(t *testing.T, ok bool, format string, args ...any) {
t.Helper()
if !ok {
t.Fatalf(format, args...)
}
}
cmd/mesh-controller/status.go
+268
View File
@@ -0,0 +1,268 @@
package main
import (
"context"
"flag"
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// is anything broken, is anything not answering, is anything out of date.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// short is a commit as a person refers to it.
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
// statusCommand answers "did my change go out?".
//
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
// this is worse to live with whatever its other properties.
//
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
// source now has, and which machines are running the old one.
func statusCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("status", flag.ContinueOnError)
asJSON := set.Bool("json", false, "the same answers, for something other than a person")
if _, err := parseAround(set, args); err != nil {
return err
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
asked, err := theThreeQuestions(ctx, open)
if err != nil {
return err
}
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
behind, sources := asked.behind, asked.sources
if *asJSON {
body, err := statusAsJSON(asked)
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
if len(asked.refused) > 0 {
// First, above everything else. A machine that cannot be worked out is not running an old
// declaration — it has no declaration, and nothing below this line is about it.
var names []string
for name := range asked.refused {
names = append(names, name)
}
sort.Strings(names)
fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
len(names))
for _, name := range names {
fmt.Printf(" %s\n", name)
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
fmt.Printf(" %s\n", strings.TrimSpace(line))
}
}
fmt.Println()
}
if asked.network != "" {
fmt.Printf("the private network could not be computed:\n %s\n\n",
strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
}
if len(wrong) > 0 {
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
for _, d := range wrong {
fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
if d.Refused != "" {
// The host's own words. It says exactly what it could not accept, and nothing
// written here would say it better.
fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused))
}
for _, f := range d.Failed {
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
}
}
fmt.Println()
}
if len(quiet) > 0 {
var said []string
for _, n := range quiet {
said = append(said, n.Name+" ("+heardFrom(n)+")")
}
fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n",
len(quiet), strings.Join(said, "\n "))
}
if len(behind) > 0 {
var names []string
for m := range behind {
names = append(names, m)
}
sort.Strings(names)
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
for _, m := range names {
from := sources[m]
fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
if on := behind[m]; len(on) > 0 {
// The part somebody actually wants. A module being out of date is a fact about
// the catalogue; machines running the old one is the thing with consequences.
fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", "))
} else {
fmt.Printf(" %-18s assigned to nothing\n", "")
}
}
// The remedy, beside the problem. A status that says what is wrong and not what to do
// about it makes somebody go and find the command, and the command is the whole point of
// having noticed.
fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n")
fmt.Println()
}
if len(asked.waiting) > 0 {
// The other half of "is anything out of date": a module behind its source says the
// catalogue is old, and this says a machine is — and only this one has somebody's change
// waiting inside it.
var told, never []string
for _, m := range asked.waiting {
if m.Never {
never = append(never, m.Node)
continue
}
told = append(told, m.Node)
}
if len(told) > 0 {
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
len(told), strings.Join(told, ", "))
}
if len(never) > 0 {
// Never told is not out of date. The remedy is the same push and the situation is
// not the same at all: nobody has ever asked this machine to be anything.
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
len(never), strings.Join(never, ", "))
}
fmt.Printf("\n `push --behind` sends them\n\n")
}
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
len(asked.refused) == 0 && asked.network == "" {
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
// and getting here means every question was asked and answered.
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
"the mesh would send them, and every module current with its source\n", len(nodes))
}
return nil
}
// firstLine is as much of a failure as belongs in a list.
func firstLine(s string) string {
if cut := strings.IndexByte(s, '\n'); cut >= 0 {
return strings.TrimSpace(s[:cut])
}
return strings.TrimSpace(s)
}
// builds keeps what a builder said, for the serving control plane.
//
// A type of its own rather than a method on the enrolment, because they are unrelated things
// arriving on one queue and an implementation of one should not have to say anything about the
// other.
type builds struct{ inv *inventory.Inventory }
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
//
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
// are three now — a person's status, its JSON, and a page — and three implementations of "which
// machine is not doing what it was told" would be three chances to disagree about it.
//
// The order is the design and not a convenience: is anything broken, is anything not answering, is
// anything out of date. The first has consequences now, the second may, the third is a plan for
// later — and anything that led with the third would bury the first.
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
inv := open.inventory
var out answers
var err error
out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
if err != nil {
return answers{}, err
}
out.nodes, err = inv.Nodes(ctx)
if err != nil {
return answers{}, err
}
for _, n := range out.nodes {
// Never heard from, or not lately. Different from failing: a machine that says nothing
// may be new, switched off, or unreachable, and none of those is a machine that tried
// and could not.
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
out.quiet = append(out.quiet, n)
}
}
out.behind, err = inv.Behind(ctx)
if err != nil {
return answers{}, err
}
// And why any machine cannot be worked out at all, which is neither of the first two questions
// and is asked before them both in practice: a machine nothing can be computed for is not
// broken, not quiet and not behind, and every other answer here would call it well.
//
// Read through whoResolves, which is what the private network is built from, so this and the
// network agree about who could not be resolved rather than deciding it twice.
_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
if err != nil {
return answers{}, err
}
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
// this one says a machine is — and only the second has anybody's change waiting in it.
//
// **One machine that cannot be resolved must not take the answer away from every other**
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
// reason is kept and reported as data; every question that does not depend on it is still
// answered.
would, err := wouldSend(ctx, open, out.nodes)
if err != nil {
out.network = err.Error()
would = map[string]string{}
}
out.waiting, err = inv.Waiting(ctx, would)
if err != nil {
return answers{}, err
}
out.reported, err = inv.LastReports(ctx)
if err != nil {
return answers{}, err
}
out.sources = map[string]inventory.Source{}
for module := range out.behind {
from, err := inv.SourceOf(ctx, module)
if err != nil {
return answers{}, err
}
out.sources[module] = from
}
return out, nil
}
cmd/mesh-controller/status_test.go
+121
View File
@@ -0,0 +1,121 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
// **One machine's failure must never take the answer away from a reader asking about the others.**
//
// A blocked machine is not on the private network, and a mesh whose hub is that machine has no hub
// — which came back through the reading as a refusal, so `status` printed nothing at all and
// `status --json` emitted multi-line prose on stderr and not one byte of JSON. A machine-readable
// interface that stops being machine-readable exactly when something is wrong is one nobody can
// build an alarm on.
func TestOneBlockedMachineDoesNotDestroyTheWholeDocument(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
// Break the hub, using nothing but the command a person has.
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{"rival-one", "rival-two"} {
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
t.Fatal(err)
}
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatalf("one blocked machine made the whole mesh unreadable: %v", err)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var parsed map[string]any
if err := json.Unmarshal(body, &parsed); err != nil {
t.Fatalf("what a script would read is not JSON: %v\n%s", err, body)
}
// The failure is in the document, as data, on the machine it belongs to.
unresolved, _ := parsed["unresolved"].([]any)
if len(unresolved) == 0 {
t.Fatalf("a machine that cannot be worked out is absent from the document:\n%s", body)
}
var found bool
for _, row := range unresolved {
entry, _ := row.(map[string]any)
if entry["node"] != "anchor" {
continue
}
found = true
problem, _ := entry["problem"].(string)
if !strings.Contains(problem, "the-seat") {
t.Errorf("the machine's problem is not its own words: %q", problem)
}
}
if !found {
t.Fatalf("the blocked machine is not the one named:\n%s", body)
}
// And the mesh is still counted, so a reader can tell "none blocked" from "none at all".
if parsed["machines"] != float64(2) {
t.Errorf("the rest of the document did not survive: %v", parsed["machines"])
}
}
// A well mesh carries the field as an empty list, not as nothing: a reader distinguishing "none
// blocked" from "this field is missing" would have to handle both, and null is the one that gets
// forgotten.
func TestAWellMeshCarriesAnEmptyUnresolvedList(t *testing.T) {
open := aMesh(t)
asked, err := theThreeQuestions(t.Context(), open)
if err != nil {
t.Fatal(err)
}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var parsed map[string]any
if err := json.Unmarshal(body, &parsed); err != nil {
t.Fatal(err)
}
list, ok := parsed["unresolved"].([]any)
if !ok {
t.Fatalf("\"unresolved\" is %T, not a list:\n%s", parsed["unresolved"], body)
}
if len(list) != 0 {
t.Fatalf("a well mesh reports blocked machines: %v", list)
}
if _, said := parsed["network"]; said {
t.Errorf("a well mesh says the network could not be computed: %v", parsed["network"])
}
}
// And the page says it too, from the same reading. A board that renders "all well" over a mesh
// where nothing can be sent anywhere is worse than a board that is down.
func TestThePageSaysWhichMachinesCannotBeWorkedOut(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
one, two := rivals()
register(t, open, one)
register(t, open, two)
for _, m := range []string{"rival-one", "rival-two"} {
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
t.Fatal(err)
}
}
asked, err := theThreeQuestions(ctx, open)
if err != nil {
t.Fatal(err)
}
rendered := render(t, viewOf(asked))
for _, want := range []string{"Can everything be worked out?", "anchor", "the-seat"} {
if !strings.Contains(rendered, want) {
t.Fatalf("the page does not say %q:\n%s", want, rendered)
}
}
}
cmd/mesh-controller/stores.go
+178
View File
@@ -0,0 +1,178 @@
package main
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/store"
)
// reaching each context's store, which no other context may touch.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
// migrate brings every held context's schema up to date.
//
// Reported per context and per migration, because this runs during a bootstrap on a machine with
// nothing else on it — the output is the only account of what happened, and "migrated" is not one.
func migrate(ctx context.Context) error {
for _, c := range held {
migrations, err := c.migrations()
if err != nil {
return err
}
s, err := store.Open(ctx, c.name)
if err != nil {
return err
}
defer s.Close()
// The bootstrap raises PostgreSQL moments before this runs, and a container that is
// running is not a database that will answer — a distinction this project has already
// paid for once, when a crash-looping database reported itself as up between restarts.
if err := s.Ready(ctx, 60*time.Second); err != nil {
return err
}
done, err := s.Migrate(ctx, migrations)
for _, m := range done {
fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name)
}
if err != nil {
return err
}
if len(done) == 0 {
applied, err := s.AppliedMigrations(ctx)
if err != nil {
return err
}
fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied))
}
}
// The modules the control plane ships with itself. Recorded here rather than by hand, because
// a mesh whose own private network is missing from the catalogue would have nothing to assign
// and no way to say why.
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
for _, m := range provided {
if err := inv.Provide(ctx, m); err != nil {
return err
}
fmt.Printf("provided %s\n", m.Module)
}
return nil
}
// openInventory connects and waits, the way every command that touches it needs to.
func openInventory(ctx context.Context) (*inventory.Inventory, error) {
inv, err := inventory.Open(ctx)
if err != nil {
return nil, err
}
if err := inv.Ready(ctx, 30*time.Second); err != nil {
inv.Close()
return nil, err
}
return inv, nil
}
func openIdentity(ctx context.Context) (*identity.Identity, error) {
ident, err := identity.Open(ctx)
if err != nil {
return nil, err
}
if err := ident.Ready(ctx, 30*time.Second); err != nil {
ident.Close()
return nil, err
}
return ident, nil
}
// openLicences connects to the context that holds which model access exists and who may use it.
func openLicences(ctx context.Context) (*licences.Licences, error) {
held, err := licences.Open(ctx)
if err != nil {
return nil, err
}
if err := held.Ready(ctx, 30*time.Second); err != nil {
held.Close()
return nil, err
}
return held, nil
}
// stores is what one command has open.
//
// **Opened once, not once per machine.** Working out what a machine should be reaches the identity
// context for its certificate and the licence context for its model access, and both were opened —
// and waited on — inside functions called for every node in a push. Two machines hid it; fifty
// would be fifty connect-and-wait cycles for data that does not change while the push runs.
//
// Each is opened on first use rather than up front, because most commands need one context and
// paying to reach three would be the same waste from the other side.
type stores struct {
inventory *inventory.Inventory
identity *identity.Identity
licences *licences.Licences
}
// open connects to the inventory, which every command that touches the mesh needs.
func openStores(ctx context.Context) (*stores, error) {
inv, err := openInventory(ctx)
if err != nil {
return nil, err
}
return &stores{inventory: inv}, nil
}
// Identity is this control plane's own identity context, opened if it has not been.
func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
if h.identity != nil {
return h.identity, nil
}
opened, err := openIdentity(ctx)
if err != nil {
return nil, err
}
h.identity = opened
return opened, nil
}
// Licences is the context holding model access, opened if it has not been.
func (h *stores) Licences(ctx context.Context) (*licences.Licences, error) {
if h.licences != nil {
return h.licences, nil
}
opened, err := openLicences(ctx)
if err != nil {
return nil, err
}
h.licences = opened
return opened, nil
}
// Close lets go of everything that was opened, in any order: they are separate connections to
// separate databases and none of them knows about the others.
func (h *stores) Close() {
if h.licences != nil {
h.licences.Close()
}
if h.identity != nil {
h.identity.Close()
}
if h.inventory != nil {
h.inventory.Close()
}
}
cmd/mesh-controller/upgrades.go
+196
View File
@@ -0,0 +1,196 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// following acts on what the catalogue announces.
//
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
// the control plane.
type following struct{ open *stores }
// Upgraded sends the machines running a module the version the catalogue now considers current —
// or records that they are behind, which is the default and needs no record.
//
// **Recording is not a second code path.** A machine that is not running what the mesh would send
// it is already something the mesh notices and reports; that is what `status` and `push --behind`
// are built on. So "record it" is the absence of an action, and the only thing this has to decide
// is whether to act.
func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
inv := f.open.inventory
decision, err := inv.UpgradeOf(ctx, u.Module)
if err != nil {
return err
}
on, err := inv.Running(ctx, u.Module)
if err != nil {
return err
}
if len(on) == 0 {
fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit))
return nil
}
if !decision.RollOut {
// Named rather than counted, and said even though nothing happens: an upgrade that was
// deliberately not rolled out and an upgrade that was never noticed look identical in a
// log that only speaks when it acts.
fmt.Printf("%s moved to %s; %s %s behind it, and this mesh records upgrades rather than "+
"rolling them out — `push --behind` when you want them\n",
u.Module, shortCommit(u.Commit), readableList(on), isAre(len(on)))
return nil
}
if decision.Together {
fmt.Printf("%s moved to %s; sending %s together\n",
u.Module, shortCommit(u.Commit), readableList(on))
return sendTo(ctx, f.open, on)
}
// One at a time, and stopping at the first that fails.
//
// **Stopping is the point.** The machines are done one after another precisely so that a
// version that breaks the first one does not reach the rest; carrying on past a failure would
// make this the same as sending them together, only slower.
fmt.Printf("%s moved to %s; sending %s one at a time\n",
u.Module, shortCommit(u.Commit), readableList(on))
for _, node := range on {
if err := sendTo(ctx, f.open, []string{node}); err != nil {
return fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
node, u.Module, err)
}
}
return nil
}
// readableList names machines the way a sentence does, because this is read by a person deciding
// whether an upgrade went where they expected.
func readableList(names []string) string {
switch len(names) {
case 0:
return "nothing"
case 1:
return names[0]
case 2:
return names[0] + " and " + names[1]
}
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
}
func shortCommit(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
func isAre(n int) string {
if n == 1 {
return "is"
}
return "are"
}
// upgradeCommand says what should happen when a module's current version moves.
func upgradeCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
together := set.Bool("together", false,
"send every machine running it at once, instead of one after another")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) == 0 {
return errors.New("upgrade <module> [roll-out|record] [--together]")
}
module := positionals[0]
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
if len(positionals) == 1 {
decision, err := inv.UpgradeOf(ctx, module)
if err != nil {
return err
}
fmt.Println(sayUpgrade(module, decision))
return nil
}
var decision inventory.Upgrade
switch positionals[1] {
case "roll-out":
decision = inventory.Upgrade{RollOut: true, Together: *together}
case "record":
if *together {
// Refused rather than ignored: --together only means anything for a roll-out, and
// accepting it here would store a preference that never applies and looks like it does.
return errors.New("`--together` says how to roll out, so it cannot be given with " +
"`record`, which is the choice not to")
}
decision = inventory.Upgrade{}
default:
return fmt.Errorf("upgrade <module> roll-out|record — not %q", positionals[1])
}
if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
return err
}
fmt.Println(sayUpgrade(module, decision))
return nil
}
func sayUpgrade(module string, u inventory.Upgrade) string {
if !u.RollOut {
return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
"reported as behind", module)
}
if u.Together {
return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
"together", module)
}
return fmt.Sprintf("when %s moves, the machines running it are sent the new version one at "+
"a time, stopping at the first that fails", module)
}
// Announceable is every build this mesh recorded, in the shape the builder announces one.
//
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
// (novox/hq 04-ISSUES/050). A durable queue keeps what arrived after it existed, so a running
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
// store the catalogue runs on, and the catalogue itself.
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
builds, err := f.open.inventory.Announceable(ctx)
if err != nil {
return nil, err
}
out := make([]link.Announcement, 0, len(builds))
for _, b := range builds {
a := link.Announcement{
Module: b.Module, Commit: b.Commit, Repository: b.Repository,
Path: b.Path, Ref: b.Ref, Against: b.Against,
}
if len(b.Manifest) > 0 {
a.Manifest = b.Manifest
}
for _, made := range b.Made {
a.Made = append(a.Made, link.MadeArtifact{
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
})
}
out = append(out, a)
}
return out, nil
}