A module is a repository and a path, the builder announces what it made, and the mesh acts on it #20
@@ -183,6 +183,7 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
|||||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
result.Against = built.Against
|
||||||
fmt.Printf(" built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
fmt.Printf(" built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -211,11 +212,45 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
|
fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err)
|
||||||
}
|
}
|
||||||
|
// **And announced, which is a different act from answering.** The reply goes to whoever asked
|
||||||
|
// and is correlated to their request; this says to the whole mesh that a module now exists at
|
||||||
|
// a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build
|
||||||
|
// nobody asked for still has to be announced, or the graph knows less than the registry does.
|
||||||
|
//
|
||||||
|
// Only on success: a failed build produced no module-version, and announcing one would put
|
||||||
|
// something in the graph that was never made.
|
||||||
|
if result.Failed == "" && result.Commit != "" {
|
||||||
|
announced := map[string]any{
|
||||||
|
"module": moduleOf(result.Manifest), "commit": result.Commit,
|
||||||
|
"repository": result.Repository, "path": result.Path, "ref": result.Ref,
|
||||||
|
"manifest": json.RawMessage(result.Manifest), "against": result.Against,
|
||||||
|
"made": result.Made,
|
||||||
|
}
|
||||||
|
if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil {
|
||||||
|
// Said, not fatal: the build happened and was answered. A module the catalogue has not
|
||||||
|
// heard of is a gap somebody can close; a build reported as failed because announcing
|
||||||
|
// it failed is a lie about work that was done.
|
||||||
|
fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
|
// Acknowledged only once the answer is away, so a builder that dies before answering leaves
|
||||||
// the request for another machine rather than losing it.
|
// the request for another machine rather than losing it.
|
||||||
_ = delivery.Ack(false)
|
_ = delivery.Ack(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// moduleOf reads the module's name out of the manifest it just built, which is the only place it is
|
||||||
|
// authoritative — the request named a repository and a path, not a module.
|
||||||
|
func moduleOf(manifest json.RawMessage) string {
|
||||||
|
var named struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(manifest, &named); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return named.Module
|
||||||
|
}
|
||||||
|
|
||||||
func short(commit string) string {
|
func short(commit string) string {
|
||||||
if len(commit) > 8 {
|
if len(commit) > 8 {
|
||||||
return commit[:8]
|
return commit[:8]
|
||||||
|
|||||||
@@ -242,11 +242,14 @@ func (m *Management) CreateBuilderAccount(ctx context.Context, name, password st
|
|||||||
// and a queue nobody may declare is a queue that exists only if the control plane has
|
// and a queue nobody may declare is a queue that exists only if the control plane has
|
||||||
// already run, which makes the order they start in matter.
|
// already run, which makes the order they start in matter.
|
||||||
"configure": "^" + builds + "$",
|
"configure": "^" + builds + "$",
|
||||||
// The exchange, and nothing else. **Not the default exchange**: permission there is
|
// Two exchanges, and nothing else. **Not the default exchange**: permission there is
|
||||||
// granted per exchange rather than per queue, so a builder allowed to use it could
|
// granted per exchange rather than per queue, so a builder allowed to use it could
|
||||||
// publish into any node's queue — the privilege a build machine most obviously should
|
// publish into any node's queue — the privilege a build machine most obviously should
|
||||||
// not have. Answers go through the exchange, which is why they can.
|
// not have. Answers go through the node exchange; announcing what was built goes through
|
||||||
"write": "^" + regexp.QuoteMeta(ExchangeName) + "$",
|
// the events exchange, which is a different act with a different audience (novox/hq
|
||||||
|
// ADR 0072). A builder that could answer and not announce would leave the module graph
|
||||||
|
// knowing less than the registry does.
|
||||||
|
"write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + regexp.QuoteMeta(EventsExchangeName) + ")$",
|
||||||
// The build queue and nothing else. Not another machine's declarations.
|
// The build queue and nothing else. Not another machine's declarations.
|
||||||
"read": "^" + builds + "$",
|
"read": "^" + builds + "$",
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -42,6 +43,15 @@ type Publisher interface {
|
|||||||
|
|
||||||
// Result is everything one build produced.
|
// Result is everything one build produced.
|
||||||
type Result struct {
|
type Result struct {
|
||||||
|
// Against is every pinned image this build was built on top of, read out of its own inputs.
|
||||||
|
//
|
||||||
|
// **Derived, not declared** (novox/hq ADR 0009): a declared list of dependencies drifts from
|
||||||
|
// what the code actually uses, and an artifact is out of date when anything it was built
|
||||||
|
// against moved. These are artifact references rather than module-versions, because that is
|
||||||
|
// what a build input names; resolving them to modules is the catalogue's work, since it is
|
||||||
|
// what knows which module-version published which artifact.
|
||||||
|
Against []string
|
||||||
|
|
||||||
// Manifest is the module as the mesh should hold it: artifacts resolved to digests.
|
// Manifest is the module as the mesh should hold it: artifacts resolved to digests.
|
||||||
Manifest catalogue.Manifest
|
Manifest catalogue.Manifest
|
||||||
// Commit is what was built, so "is this current?" is answerable without building again.
|
// Commit is what was built, so "is this current?" is answerable without building again.
|
||||||
@@ -124,7 +134,8 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
return Result{Manifest: resolved, Commit: commit, Built: built}, nil
|
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||||
|
Against: against(within, manifest)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// inside resolves a module's path within a clone, and refuses one that leaves it.
|
// inside resolves a module's path within a clone, and refuses one that leaves it.
|
||||||
@@ -157,6 +168,39 @@ func describe(path string) string {
|
|||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pinnedImage matches an image reference pinned by digest, which is the only kind a build input is
|
||||||
|
// allowed to name — a tag is something somebody else can move under you.
|
||||||
|
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)
|
||||||
|
|
||||||
|
// against reads what this module's image artifacts are built on top of, out of the files that
|
||||||
|
// build them. Nothing is guessed: a reference that is not written down is not reported.
|
||||||
|
func against(within string, manifest catalogue.Manifest) []string {
|
||||||
|
if manifest.Build == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, a := range manifest.Build.Artifacts {
|
||||||
|
if a.Kind != catalogue.ArtifactImage || a.From == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(filepath.Join(within, a.From))
|
||||||
|
if err != nil {
|
||||||
|
// Not fatal: the build itself already failed if this file was needed and missing, and
|
||||||
|
// reporting no edges is honest where inventing them would not be.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, found := range pinnedImage.FindAllString(string(body), -1) {
|
||||||
|
if !seen[found] {
|
||||||
|
seen[found] = true
|
||||||
|
out = append(out, found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// ManifestName is the one file a module repository must have.
|
// ManifestName is the one file a module repository must have.
|
||||||
//
|
//
|
||||||
// At the root, and named the same in every repository. A convention somebody can look for beats a
|
// At the root, and named the same in every repository. A convention somebody can look for beats a
|
||||||
|
|||||||
@@ -82,6 +82,10 @@ type BuildResult struct {
|
|||||||
// Made is each artifact, for reporting.
|
// Made is each artifact, for reporting.
|
||||||
Made []MadeArtifact `json:"made,omitempty"`
|
Made []MadeArtifact `json:"made,omitempty"`
|
||||||
|
|
||||||
|
// Against is every pinned image this was built on top of, read out of the build's own inputs
|
||||||
|
// (novox/hq ADR 0009). The catalogue turns these into edges; nothing else need care.
|
||||||
|
Against []string `json:"against,omitempty"`
|
||||||
|
|
||||||
// Failed is why, when it did.
|
// Failed is why, when it did.
|
||||||
Failed string `json:"failed,omitempty"`
|
Failed string `json:"failed,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Emitting a module event from Go.
|
||||||
|
//
|
||||||
|
// **Every event rides one topic exchange** (novox/hq ADR 0042), which is not the direct exchange
|
||||||
|
// nodes and the control plane speak over. A module that announces something publishes here, and
|
||||||
|
// consumers bind their own durable queue to a pattern over it.
|
||||||
|
//
|
||||||
|
// This exists because the builder is a module written in Go while every other emitter is
|
||||||
|
// TypeScript on the sdk. The envelope is the sdk's, reproduced exactly: the body is the payload
|
||||||
|
// alone and everything about the event travels as headers. A second shape would be a second thing
|
||||||
|
// for consumers to handle, and they are written against the first.
|
||||||
|
const (
|
||||||
|
// EventsExchange is where every event rides. Named here rather than imported from the broker
|
||||||
|
// package for the same reason BuildQueueName is duplicated there — one direction of dependency.
|
||||||
|
EventsExchange = "mesh.events"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EmitEvent publishes one module event, in the envelope the sdk's consumers expect.
|
||||||
|
//
|
||||||
|
// Persistent, because an event that a broker restart loses is not an announcement. The publish is
|
||||||
|
// not confirmed here: the caller has already done the work the event describes, and a build that
|
||||||
|
// succeeded must not be reported as failed because saying so failed.
|
||||||
|
func EmitEvent(ctx context.Context, channel *amqp.Channel, eventType, source, node string, body any) error {
|
||||||
|
payload, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot serialise a %s event: %w", eventType, err)
|
||||||
|
}
|
||||||
|
id, err := eventID()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return channel.PublishWithContext(ctx, EventsExchange, eventType, false, false, amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
DeliveryMode: amqp.Persistent,
|
||||||
|
MessageId: id,
|
||||||
|
Timestamp: time.Now().UTC(),
|
||||||
|
Body: payload,
|
||||||
|
Headers: amqp.Table{
|
||||||
|
"x-event-id": id,
|
||||||
|
"x-source": source,
|
||||||
|
"x-node": node,
|
||||||
|
"x-time": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able
|
||||||
|
// to tell a redelivery from a second event, and only the emitter can say which it is.
|
||||||
|
func eventID() (string, error) {
|
||||||
|
raw := make([]byte, 16)
|
||||||
|
if _, err := rand.Read(raw); err != nil {
|
||||||
|
return "", fmt.Errorf("cannot make an event id: %w", err)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(raw), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeyModuleBuilt is what the builder announces when it has built something. The catalogue places
|
||||||
|
// it in the module graph; nothing else need care.
|
||||||
|
const KeyModuleBuilt = "module.builder.built"
|
||||||
Reference in New Issue
Block a user