The mesh builds: a machine takes the work, and the catalogue shows it
A build is work, not state. Everything else the control plane sends a node is a declaration — this is what you should be — reconciled forever. A build happens once and is finished. Putting it in a declaration would mean rebuilding on every reconcile, or a declaration carrying "and I already did this", which is state about an event rather than about a machine. So it travels on its own queue and the answer comes back correlated. One queue, so several build machines share the work and each request is done exactly once — which a per-machine routing key would not give. mesh-builder is the program a build machine runs. Not the control plane, which must not run commands on a machine; not the host, which would then need a container runtime and git everywhere to do something almost no machine will ever do. It holds its own broker credential and nothing else. Three properties that are decisions: - a request is acknowledged only once the answer is away, so a builder that dies mid-build leaves the work for another machine rather than losing it with nobody ever hearing why - one build at a time. Five at once against one runtime finishes all five slower than it would have finished the first, and the queue is what shares work between machines - a failure is a RESULT. A build that fails silently is indistinguishable from a builder that is not running, and those want different responses And `module list` is a catalogue: what exists, at which version, built from which commit or handed over by hand or shipped with the control plane, whether it is behind its source, and which machines run it. All of that was recorded from the first build and none of it was shown, so "is this current?" could only be answered by reading the database. Proven against a real broker, registry and store: the mesh asked, a builder consumed, built, published, answered; the manifest was recorded with its commit; the source moved and the catalogue said "behind"; rebuilding caught it up with a new digest because the content changed.
This commit is contained in:
@@ -515,3 +515,63 @@ func (i *Inventory) pinRows(ctx context.Context, nodeName string) (int, error) {
|
||||
`select count(*) from provision_pin where node = $1`, node.ID).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Entry is one module as a catalogue shows it: what it is, where it came from, and who runs it.
|
||||
type Entry struct {
|
||||
Manifest catalogue.Manifest
|
||||
Source Source
|
||||
// On is every node this module is assigned to, sorted.
|
||||
On []string
|
||||
// Provided is true when the module came with the control plane rather than from a repository.
|
||||
Provided bool
|
||||
}
|
||||
|
||||
// Catalogued is every module the mesh knows about, with everything a person asks about one.
|
||||
//
|
||||
// **One query rather than a call per module.** A catalogue that costs a round trip per row is a
|
||||
// catalogue nobody lists, and the questions here — what is this, where did it come from, who is
|
||||
// running it — are asked together every time.
|
||||
func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select m.name, m.manifest,
|
||||
coalesce(m.source, ''), coalesce(m.ref, ''),
|
||||
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
|
||||
coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}')
|
||||
from module m
|
||||
left join assignment a on a.module = m.name
|
||||
left join node n on n.id = a.node
|
||||
group by m.name, m.manifest, m.source, m.ref, m.built_from, m.source_head
|
||||
order by m.name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Entry
|
||||
for rows.Next() {
|
||||
var raw []byte
|
||||
var name string
|
||||
var source Source
|
||||
var on []string
|
||||
if err := rows.Scan(&name, &raw, &source.Repository, &source.Ref,
|
||||
&source.BuiltFrom, &source.Head, &on); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry := Entry{Manifest: m, Source: source, On: on}
|
||||
if source.Repository == providedBy {
|
||||
// It came with the control plane. Not a repository, and showing it as one would have
|
||||
// somebody go looking for it.
|
||||
entry.Provided = true
|
||||
entry.Source = Source{}
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// providedBy is what the source column says for a module the control plane ships.
|
||||
const providedBy = "the control plane"
|
||||
|
||||
@@ -3,6 +3,7 @@ package inventory
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-control/internal/catalogue"
|
||||
@@ -428,3 +429,91 @@ func TestAPinGoesWhenTheProviderLeavesTheMesh(t *testing.T) {
|
||||
t.Fatalf("a choice outlived the machine it named: %d row(s) left", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCatalogueSaysWhereEachModuleCameFromAndWhoRunsIt(t *testing.T) {
|
||||
// The provenance was recorded from the first build and nothing showed it, which made "is this
|
||||
// current?" a question you could only answer by reading the database.
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
for _, n := range []string{"workstation", "laptop"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest("shell", []string{"login-shell"}, nil),
|
||||
Source{Repository: "https://forge.invalid/shell.git", BuiltFrom: "aaa", Head: "aaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest("byhand", nil, nil), Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.Provide(ctx, manifest("networking", nil, []string{"login-shell"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"workstation", "laptop"} {
|
||||
if err := inv.Assign(ctx, n, "shell"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]Entry{}
|
||||
for _, e := range entries {
|
||||
by[e.Manifest.Module] = e
|
||||
}
|
||||
if len(by) != 3 {
|
||||
t.Fatalf("the catalogue has %d modules", len(by))
|
||||
}
|
||||
|
||||
// Sorted, and both nodes, so a person reading it twice sees the same thing.
|
||||
if got := strings.Join(by["shell"].On, ","); got != "laptop,workstation" {
|
||||
t.Fatalf("shell runs on %q", got)
|
||||
}
|
||||
if by["shell"].Source.Repository != "https://forge.invalid/shell.git" {
|
||||
t.Fatalf("shell came from %q", by["shell"].Source.Repository)
|
||||
}
|
||||
if by["shell"].Provided {
|
||||
t.Fatal("a module built from a repository was reported as shipped with the control plane")
|
||||
}
|
||||
|
||||
// A module nobody runs is in the catalogue: the catalogue is what EXISTS, and what runs is a
|
||||
// different question the same row answers.
|
||||
if len(by["byhand"].On) != 0 {
|
||||
t.Fatalf("byhand runs on %v", by["byhand"].On)
|
||||
}
|
||||
// Handed over by hand is its own state. Nothing can rebuild it, and showing it as a
|
||||
// repository would send somebody looking for one.
|
||||
if by["byhand"].Source.Repository != "" || by["byhand"].Provided {
|
||||
t.Fatalf("byhand: %+v", by["byhand"])
|
||||
}
|
||||
|
||||
if !by["networking"].Provided {
|
||||
t.Fatal("a module the control plane ships was not marked as such")
|
||||
}
|
||||
if by["networking"].Source.Repository != "" {
|
||||
// It is not a repository, and showing it as one would have somebody go looking for it.
|
||||
t.Fatalf("networking claims to come from %q", by["networking"].Source.Repository)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACatalogueEntryKnowsWhetherItIsBehind(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
if err := inv.RegisterModule(ctx, manifest("shell", nil, nil),
|
||||
Source{Repository: "https://forge.invalid/shell.git", BuiltFrom: "aaa", Head: "aaa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SourceMoved(ctx, "shell", "bbb"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entries[0].Source.Current() {
|
||||
t.Fatal("a module whose source moved reported itself current")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
// Asking a machine to build a module, and hearing what came out.
|
||||
//
|
||||
// **A build is work, not state.** Everything else the control plane sends a node is a declaration
|
||||
// — *this is what you should be* — and the node reconciles toward it forever. A build happens
|
||||
// once, produces something, and is finished. Putting it in a declaration would mean a machine
|
||||
// rebuilding on every reconcile, or the declaration carrying "and I already did this", which is
|
||||
// state about an event rather than about the machine.
|
||||
//
|
||||
// So it travels on its own queue, and the reply comes back correlated. That is also why the
|
||||
// builder is a **separate consumer** rather than the host: the host applies declarations and
|
||||
// holds no opinion about what they contain, and a host that also built things would be a host
|
||||
// with a container runtime requirement and a git dependency (novox/hq ADR 0005).
|
||||
|
||||
// BuildQueue is where build requests wait. One queue, so several build machines can share the
|
||||
// work and each request is done exactly once — which is what a queue is for and what a
|
||||
// per-machine routing key would not give.
|
||||
const BuildQueue = "builds"
|
||||
|
||||
// KeyBuilt is what a builder publishes when it has finished, successfully or not.
|
||||
const KeyBuilt = "built"
|
||||
|
||||
// BuildRequest is one module to build.
|
||||
type BuildRequest struct {
|
||||
// ID correlates the answer with the asking. Not the module name: two builds of one module can
|
||||
// be in flight, and the second answer is not the first one's.
|
||||
ID string `json:"id"`
|
||||
// Repository is where the source is, as git would clone it.
|
||||
Repository string `json:"repository"`
|
||||
// Ref is the branch, tag or commit. Empty means whatever the repository's default is, which
|
||||
// is the only case where the mesh does not know what it built until it has built it.
|
||||
Ref string `json:"ref,omitempty"`
|
||||
}
|
||||
|
||||
// BuildResult is what a builder says back.
|
||||
//
|
||||
// **Failure is a result, not an absence.** A build that fails and says nothing is
|
||||
// indistinguishable from a builder that is not running, and those want completely different
|
||||
// responses — the same rule the host follows about a service that does not exist.
|
||||
type BuildResult struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
|
||||
// On is the machine that did it, so a failure that is about one machine can be told from one
|
||||
// about the source.
|
||||
On string `json:"on"`
|
||||
|
||||
// Commit is what was actually built. The mesh records it, which is what makes "is this
|
||||
// current?" answerable without building again.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
|
||||
// Manifest is the module as the mesh should hold it, artifacts resolved to digests. Raw,
|
||||
// because the control plane parses it with the same parser it uses for one handed over by
|
||||
// hand — a second path would be a second thing to disagree.
|
||||
Manifest json.RawMessage `json:"manifest,omitempty"`
|
||||
|
||||
// Made is each artifact, for reporting.
|
||||
Made []MadeArtifact `json:"made,omitempty"`
|
||||
|
||||
// Failed is why, when it did.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
// MadeArtifact is one thing a build produced, as a person would want it reported.
|
||||
type MadeArtifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
// RequestBuild asks for a module to be built and waits for the answer.
|
||||
//
|
||||
// Waiting rather than returning immediately, because the thing a person wants after asking for a
|
||||
// build is to know whether it worked. A build that is dispatched and forgotten needs somewhere to
|
||||
// look afterwards, and there is nowhere yet.
|
||||
func RequestBuild(ctx context.Context, channel *amqp.Channel, request BuildRequest,
|
||||
timeout time.Duration) (BuildResult, error) {
|
||||
|
||||
// Its own queue for the answer, declared before the ask. Consuming from the shared exchange
|
||||
// would mean competing with the control plane's own consumer for a message meant for this
|
||||
// caller — which is the fault this package's own doc comment records having had.
|
||||
replies, err := channel.QueueDeclare("", false, true, true, false, nil)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
answers, err := channel.ConsumeWithContext(ctx, replies.Name, "", true, true, false, false, nil)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
body, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if err := channel.PublishWithContext(ctx, "", BuildQueue, false, false, amqp.Publishing{
|
||||
ContentType: "application/json",
|
||||
DeliveryMode: amqp.Persistent,
|
||||
CorrelationId: request.ID,
|
||||
ReplyTo: replies.Name,
|
||||
Body: body,
|
||||
}); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
waiting, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case <-waiting.Done():
|
||||
return BuildResult{}, fmt.Errorf(
|
||||
"no builder answered within %s. Something must be consuming %q, and nothing is "+
|
||||
"— or it is building something that takes longer than this",
|
||||
timeout, BuildQueue)
|
||||
case delivery, ok := <-answers:
|
||||
if !ok {
|
||||
return BuildResult{}, fmt.Errorf("the connection closed while waiting for a build")
|
||||
}
|
||||
var result BuildResult
|
||||
if err := json.Unmarshal(delivery.Body, &result); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("a builder answered with something unreadable: %w", err)
|
||||
}
|
||||
if result.ID != request.ID {
|
||||
// Somebody else's answer on this queue. Ignored rather than returned, because
|
||||
// returning it would attribute one build's outcome to another's.
|
||||
continue
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user