novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

The mesh builds: a machine takes the work, and the catalogue shows it

Browse Source 
A build is work, not state. Everything else the control plane sends a
node is a declaration — this is what you should be — reconciled forever.
A build happens once and is finished. Putting it in a declaration would
mean rebuilding on every reconcile, or a declaration carrying "and I
already did this", which is state about an event rather than about a
machine.

So it travels on its own queue and the answer comes back correlated. One
queue, so several build machines share the work and each request is done
exactly once — which a per-machine routing key would not give.

mesh-builder is the program a build machine runs. Not the control plane,
which must not run commands on a machine; not the host, which would then
need a container runtime and git everywhere to do something almost no
machine will ever do. It holds its own broker credential and nothing
else.

Three properties that are decisions:

- a request is acknowledged only once the answer is away, so a builder
  that dies mid-build leaves the work for another machine rather than
  losing it with nobody ever hearing why
- one build at a time. Five at once against one runtime finishes all five
  slower than it would have finished the first, and the queue is what
  shares work between machines
- a failure is a RESULT. A build that fails silently is
  indistinguishable from a builder that is not running, and those want
  different responses

And `module list` is a catalogue: what exists, at which version, built
from which commit or handed over by hand or shipped with the control
plane, whether it is behind its source, and which machines run it. All of
that was recorded from the first build and none of it was shown, so "is
this current?" could only be answered by reading the database.

Proven against a real broker, registry and store: the mesh asked, a
builder consumed, built, published, answered; the manifest was recorded
with its commit; the source moved and the catalogue said "behind";
rebuilding caught it up with a new digest because the content changed.
This commit is contained in:
jschoubben
2026-08-30 03:46:02 +02:00
parent 7d033ad9f6
commit 421fe73dce
5 changed files with 606 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-builder/main.go
+217
View File
@@ -0,0 +1,217 @@
// mesh-builder — the thing a build machine runs.
//
// It takes work from the mesh, turns a repository into artifacts, publishes them, and says what
// came out. It is **not** the control plane and it is **not** the host:
//
// - the control plane decides and never touches a machine. Building runs commands on one, and
// what the control plane may send a machine is bounded by the declaration language
// (novox/hq ADR 0005). "Run this build" is not in it, and widening the language so it could
// be would make the control plane able to run anything anywhere.
// - the host applies declarations and holds no opinion about what they contain. A host that
// also built things would need a container runtime and git, on every machine, to do something
// almost none of them will ever do.
//
// So it is a module: a program a machine runs because the mesh told it to, holding its own broker
// credential and nothing else. Compromise of a build machine is compromise of a build machine.
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"github.com/novox/mesh-control/internal/builder"
"github.com/novox/mesh-control/internal/link"
)
// version is set at build time.
var version = "development"
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "mesh-builder: %v\n", err)
os.Exit(1)
}
}
const usage = `mesh-builder — builds modules for the mesh
It consumes build requests and answers with what it made. Nothing is listened on and nothing
is dialled except the broker.
MESH_BROKER_AMQP where the broker is, with this builder's own credential
MESH_REGISTRY host:port to publish artifacts to
MESH_WORKSPACE where to clone and build (default: a temporary directory)
`
func run() error {
if len(os.Args) > 1 {
switch os.Args[1] {
case "version":
fmt.Println(version)
return nil
default:
fmt.Print(usage)
return nil
}
}
amqpURL := strings.TrimSpace(os.Getenv("MESH_BROKER_AMQP"))
if amqpURL == "" {
return fmt.Errorf("no MESH_BROKER_AMQP: a builder with no broker has nothing to build")
}
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
if registry == "" {
return fmt.Errorf(
"no MESH_REGISTRY: a built artifact nobody can fetch is not built")
}
workspace := os.Getenv("MESH_WORKSPACE")
if workspace == "" {
workspace = os.TempDir() + "/mesh-builder"
}
on, err := os.Hostname()
if err != nil {
on = "a build machine"
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
conn, err := amqp.Dial(amqpURL)
if err != nil {
// Not quoted back: the URL carries this builder's broker password.
return fmt.Errorf("cannot reach the broker: %w", err)
}
defer conn.Close()
channel, err := conn.Channel()
if err != nil {
return err
}
defer channel.Close()
if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil {
return err
}
// One at a time. A build machine that took five requests at once would run five container
// builds against one runtime and finish all of them slower than it would have finished the
// first — and the queue is what shares work between machines, so nothing is lost by it.
if err := channel.Qos(1, 0, false); err != nil {
return err
}
// Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this
// process dies mid-way, with nobody waiting on it ever hearing why.
requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder",
false, false, false, false, nil)
if err != nil {
return err
}
fmt.Printf("building for the mesh, publishing to %s\n", registry)
publisher := builder.Registry{Address: registry, Run: builder.Command}
for {
select {
case <-ctx.Done():
fmt.Println("stopping")
return nil
case delivery, ok := <-requests:
if !ok {
return fmt.Errorf("the broker closed the connection")
}
answer(ctx, channel, publisher, on, workspace, delivery)
}
}
}
// answer does one build and says what happened, whichever way it went.
func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher,
on, workspace string, delivery amqp.Delivery) {
var request link.BuildRequest
if err := json.Unmarshal(delivery.Body, &request); err != nil {
// Unreadable. Acknowledged and dropped rather than requeued: a message this builder
// cannot parse will not become parseable by being delivered again, and requeueing it
// would put it in front of every real request for ever.
fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err)
_ = delivery.Ack(false)
return
}
result := link.BuildResult{
ID: request.ID, Repository: request.Repository, Ref: request.Ref, On: on,
}
fmt.Printf("building %s", request.Repository)
if request.Ref != "" {
fmt.Printf(" at %s", request.Ref)
}
fmt.Println()
built, err := builder.Build(ctx, builder.Command, publisher,
request.Repository, request.Ref, workspace)
if err != nil {
// A failure is a result. A build that fails and says nothing is indistinguishable from a
// builder that is not running, and those want completely different responses.
result.Failed = err.Error()
fmt.Fprintf(os.Stderr, " failed: %v\n", err)
} else {
manifest, marshalErr := json.Marshal(built.Manifest)
if marshalErr != nil {
result.Failed = marshalErr.Error()
} else {
result.Commit = built.Commit
result.Manifest = manifest
for _, made := range built.Built {
result.Made = append(result.Made, link.MadeArtifact{
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
})
}
fmt.Printf(" built %s from %s\n", built.Manifest.Module, short(built.Commit))
}
}
body, err := json.Marshal(result)
if err != nil {
fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err)
_ = delivery.Ack(false)
return
}
replyTo := delivery.ReplyTo
if replyTo == "" {
// Nobody is waiting. Still reported, to the exchange, so a control plane that records
// builds hears about it — a build whose outcome exists nowhere is one nobody can audit.
if err := channel.PublishWithContext(ctx, link.Exchange, link.KeyBuilt, false, false,
amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
fmt.Fprintf(os.Stderr, "cannot publish a build result: %v\n", err)
}
_ = delivery.Ack(false)
return
}
publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := channel.PublishWithContext(publishCtx, "", replyTo, false, false, amqp.Publishing{
ContentType: "application/json",
CorrelationId: result.ID,
Body: body,
}); err != nil {
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
}
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
// the request for another machine rather than losing it.
_ = delivery.Ack(false)
}
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
cmd/mesh-control/main.go
+99 -34
View File
@@ -20,7 +20,6 @@ import (
"time"
"github.com/novox/mesh-control/internal/broker"
"github.com/novox/mesh-control/internal/builder"
"github.com/novox/mesh-control/internal/catalogue"
"github.com/novox/mesh-control/internal/identity"
"github.com/novox/mesh-control/internal/inventory"
@@ -134,7 +133,7 @@ func usage() {
settings set <module> <file> what a module's config should say, for the whole mesh
settings set <module> <file> --node <n> ...or for one machine
settings clear <module> [--node <n>] take a layer away
build <repository> [--ref R] build a module from its source and record it
build <repository> [--ref R] have a build machine build it, and record what came out
pin <node> <provision> <from> which node this one gets a provision from
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
@@ -869,29 +868,62 @@ func moduleCommand(ctx context.Context, args []string) error {
return nil
case "list":
shelf, err := inv.Catalogue(ctx)
// The catalogue: what exists, where it came from, whether it is current, and who runs it.
// The provenance was recorded from the first build and nothing showed it, which made
// "is this current?" a question you could only answer by reading the database.
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
if len(shelf) == 0 {
if len(entries) == 0 {
fmt.Println("this mesh knows about no modules yet")
return nil
}
var names []string
for n := range shelf {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
m := shelf[n]
fmt.Printf("%-20s", m.Module)
if len(m.Provides) > 0 {
fmt.Printf(" provides %s", describeOffers(m.Provides))
var stale int
for _, e := range entries {
m := e.Manifest
fmt.Printf("%-18s %-8s", m.Module, m.Version)
switch {
case e.Provided:
fmt.Printf(" %-22s", "with the control plane")
case e.Source.Repository == "":
// Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and
// worth saying, because nothing can rebuild it.
fmt.Printf(" %-22s", "handed over")
case !e.Source.Current():
stale++
fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head))
default:
fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom))
}
for _, c := range m.Claims {
fmt.Printf(" claims %s/%s", c.At(), c.Name)
if len(e.On) > 0 {
fmt.Printf(" on %s", strings.Join(e.On, ", "))
} else {
fmt.Printf(" on nothing")
}
fmt.Println()
var says []string
if len(m.Provides) > 0 {
says = append(says, "provides "+describeOffers(m.Provides))
}
if len(m.Requires) > 0 {
says = append(says, "requires "+strings.Join(m.Requires, ", "))
}
for _, c := range m.Claims {
says = append(says, "claims "+c.At()+"/"+c.Name)
}
if len(m.Capabilities) > 0 {
says = append(says, "needs "+strings.Join(m.Capabilities, ", "))
}
if len(says) > 0 {
fmt.Printf(" %s\n", strings.Join(says, " · "))
}
}
if stale > 0 {
fmt.Printf("\n%d module(s) behind their source — `build <repository>` to catch up\n", stale)
}
return nil
@@ -912,7 +944,7 @@ func moduleCommand(ctx context.Context, args []string) error {
}
fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
args[1], short(from.BuiltFrom), short(from.Head))
fmt.Println(" build it and `module add` the result to catch up")
fmt.Printf(" run `build %s` to catch up\n", from.Repository)
return nil
case "forget":
@@ -1640,33 +1672,65 @@ func pinCommand(ctx context.Context, args []string, setting bool) error {
func buildCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("build", flag.ContinueOnError)
ref := set.String("ref", "", "the branch, tag or commit to build")
registry := set.String("registry", os.Getenv("MESH_REGISTRY"),
"host:port of the registry to publish to")
workspace := set.String("workspace", os.TempDir(), "where to clone and build")
wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if len(positionals) != 1 {
return errors.New("build <repository> [--ref R] [--registry host:port]")
}
if strings.TrimSpace(*registry) == "" {
return errors.New(
"no --registry and no MESH_REGISTRY: a built artifact nobody can fetch is not built")
return errors.New("build <repository> [--ref R] [--wait D] [--dry-run]")
}
publisher := builder.Registry{Address: *registry, Run: builder.Command}
result, err := builder.Build(ctx, builder.Command, publisher, positionals[0], *ref, *workspace)
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
for _, made := range result.Built {
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
// Correlated by something the control plane makes, not by the module's name: two builds of one
// module can be in flight, and the second answer is not the first one's.
request := link.BuildRequest{
ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
Repository: positionals[0],
Ref: *ref,
}
fmt.Printf("asked for %s", request.Repository)
if *ref != "" {
fmt.Printf(" at %s", *ref)
}
fmt.Println()
result, err := link.RequestBuild(ctx, server.Channel(), request, *wait)
if err != nil {
return err
}
if result.Failed != "" {
// The builder's own words. Wrapping them in something about the control plane would put
// two explanations between a person and a build log.
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
}
for _, made := range result.Made {
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
}
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
// second thing to disagree about what a manifest is.
manifest, err := catalogue.ParseManifest(result.Manifest)
if err != nil {
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
}
if *dryRun {
body, err := json.MarshalIndent(result.Manifest, "", " ")
body, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return err
}
@@ -1682,13 +1746,14 @@ func buildCommand(ctx context.Context, args []string) error {
// Recorded with where it came from, so "is this current?" is answerable without building it
// again (novox/hq ADR 0009).
if err := inv.RegisterModule(ctx, result.Manifest, inventory.Source{
Repository: positionals[0], Ref: *ref, BuiltFrom: result.Commit, Head: result.Commit,
if err := inv.RegisterModule(ctx, manifest, inventory.Source{
Repository: result.Repository, Ref: result.Ref,
BuiltFrom: result.Commit, Head: result.Commit,
}); err != nil {
return err
}
fmt.Printf("\n%s %s, built from %s\n",
result.Manifest.Module, result.Manifest.Version, short(result.Commit))
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", result.Manifest.Module)
fmt.Printf("\n%s %s, built on %s from %s\n",
manifest.Module, manifest.Version, result.On, short(result.Commit))
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
return nil
}