diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 2724459..a419ef0 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -183,6 +183,7 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis Name: made.Name, Kind: made.Kind, Reference: made.Reference, }) } + result.Against = built.Against fmt.Printf(" built %s from %s\n", built.Manifest.Module, short(built.Commit)) } } @@ -211,11 +212,45 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis }); err != nil { fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err) } + // **And announced, which is a different act from answering.** The reply goes to whoever asked + // and is correlated to their request; this says to the whole mesh that a module now exists at + // a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build + // nobody asked for still has to be announced, or the graph knows less than the registry does. + // + // Only on success: a failed build produced no module-version, and announcing one would put + // something in the graph that was never made. + if result.Failed == "" && result.Commit != "" { + announced := map[string]any{ + "module": moduleOf(result.Manifest), "commit": result.Commit, + "repository": result.Repository, "path": result.Path, "ref": result.Ref, + "manifest": json.RawMessage(result.Manifest), "against": result.Against, + "made": result.Made, + } + if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil { + // Said, not fatal: the build happened and was answered. A module the catalogue has not + // heard of is a gap somebody can close; a build reported as failed because announcing + // it failed is a lie about work that was done. + fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err) + } + } + // Acknowledged only once the answer is away, so a builder that dies before answering leaves // the request for another machine rather than losing it. _ = delivery.Ack(false) } +// moduleOf reads the module's name out of the manifest it just built, which is the only place it is +// authoritative — the request named a repository and a path, not a module. +func moduleOf(manifest json.RawMessage) string { + var named struct { + Module string `json:"module"` + } + if err := json.Unmarshal(manifest, &named); err != nil { + return "" + } + return named.Module +} + func short(commit string) string { if len(commit) > 8 { return commit[:8] diff --git a/internal/broker/management.go b/internal/broker/management.go index 5dbc3ae..0277d61 100644 --- a/internal/broker/management.go +++ b/internal/broker/management.go @@ -242,11 +242,14 @@ func (m *Management) CreateBuilderAccount(ctx context.Context, name, password st // and a queue nobody may declare is a queue that exists only if the control plane has // already run, which makes the order they start in matter. "configure": "^" + builds + "$", - // The exchange, and nothing else. **Not the default exchange**: permission there is + // Two exchanges, and nothing else. **Not the default exchange**: permission there is // granted per exchange rather than per queue, so a builder allowed to use it could // publish into any node's queue — the privilege a build machine most obviously should - // not have. Answers go through the exchange, which is why they can. - "write": "^" + regexp.QuoteMeta(ExchangeName) + "$", + // not have. Answers go through the node exchange; announcing what was built goes through + // the events exchange, which is a different act with a different audience (novox/hq + // ADR 0072). A builder that could answer and not announce would leave the module graph + // knowing less than the registry does. + "write": "^(" + regexp.QuoteMeta(ExchangeName) + "|" + regexp.QuoteMeta(EventsExchangeName) + ")$", // The build queue and nothing else. Not another machine's declarations. "read": "^" + builds + "$", }); err != nil { diff --git a/internal/builder/builder.go b/internal/builder/builder.go index bc1ba2b..ee5e9f4 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" @@ -42,6 +43,15 @@ type Publisher interface { // Result is everything one build produced. type Result struct { + // Against is every pinned image this build was built on top of, read out of its own inputs. + // + // **Derived, not declared** (novox/hq ADR 0009): a declared list of dependencies drifts from + // what the code actually uses, and an artifact is out of date when anything it was built + // against moved. These are artifact references rather than module-versions, because that is + // what a build input names; resolving them to modules is the catalogue's work, since it is + // what knows which module-version published which artifact. + Against []string + // Manifest is the module as the mesh should hold it: artifacts resolved to digests. Manifest catalogue.Manifest // Commit is what was built, so "is this current?" is answerable without building again. @@ -124,7 +134,8 @@ func Build(ctx context.Context, run Runner, publish Publisher, if err != nil { return Result{}, err } - return Result{Manifest: resolved, Commit: commit, Built: built}, nil + return Result{Manifest: resolved, Commit: commit, Built: built, + Against: against(within, manifest)}, nil } // inside resolves a module's path within a clone, and refuses one that leaves it. @@ -157,6 +168,39 @@ func describe(path string) string { return path } +// pinnedImage matches an image reference pinned by digest, which is the only kind a build input is +// allowed to name — a tag is something somebody else can move under you. +var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`) + +// against reads what this module's image artifacts are built on top of, out of the files that +// build them. Nothing is guessed: a reference that is not written down is not reported. +func against(within string, manifest catalogue.Manifest) []string { + if manifest.Build == nil { + return nil + } + seen := map[string]bool{} + var out []string + for _, a := range manifest.Build.Artifacts { + if a.Kind != catalogue.ArtifactImage || a.From == "" { + continue + } + body, err := os.ReadFile(filepath.Join(within, a.From)) + if err != nil { + // Not fatal: the build itself already failed if this file was needed and missing, and + // reporting no edges is honest where inventing them would not be. + continue + } + for _, found := range pinnedImage.FindAllString(string(body), -1) { + if !seen[found] { + seen[found] = true + out = append(out, found) + } + } + } + sort.Strings(out) + return out +} + // ManifestName is the one file a module repository must have. // // At the root, and named the same in every repository. A convention somebody can look for beats a diff --git a/internal/link/build.go b/internal/link/build.go index 2b44228..64ecd96 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -82,6 +82,10 @@ type BuildResult struct { // Made is each artifact, for reporting. Made []MadeArtifact `json:"made,omitempty"` + // Against is every pinned image this was built on top of, read out of the build's own inputs + // (novox/hq ADR 0009). The catalogue turns these into edges; nothing else need care. + Against []string `json:"against,omitempty"` + // Failed is why, when it did. Failed string `json:"failed,omitempty"` } diff --git a/internal/link/events.go b/internal/link/events.go new file mode 100644 index 0000000..d2266ff --- /dev/null +++ b/internal/link/events.go @@ -0,0 +1,72 @@ +package link + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// Emitting a module event from Go. +// +// **Every event rides one topic exchange** (novox/hq ADR 0042), which is not the direct exchange +// nodes and the control plane speak over. A module that announces something publishes here, and +// consumers bind their own durable queue to a pattern over it. +// +// This exists because the builder is a module written in Go while every other emitter is +// TypeScript on the sdk. The envelope is the sdk's, reproduced exactly: the body is the payload +// alone and everything about the event travels as headers. A second shape would be a second thing +// for consumers to handle, and they are written against the first. +const ( + // EventsExchange is where every event rides. Named here rather than imported from the broker + // package for the same reason BuildQueueName is duplicated there — one direction of dependency. + EventsExchange = "mesh.events" +) + +// EmitEvent publishes one module event, in the envelope the sdk's consumers expect. +// +// Persistent, because an event that a broker restart loses is not an announcement. The publish is +// not confirmed here: the caller has already done the work the event describes, and a build that +// succeeded must not be reported as failed because saying so failed. +func EmitEvent(ctx context.Context, channel *amqp.Channel, eventType, source, node string, body any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("cannot serialise a %s event: %w", eventType, err) + } + id, err := eventID() + if err != nil { + return err + } + return channel.PublishWithContext(ctx, EventsExchange, eventType, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: id, + Timestamp: time.Now().UTC(), + Body: payload, + Headers: amqp.Table{ + "x-event-id": id, + "x-source": source, + "x-node": node, + "x-time": time.Now().UTC().Format(time.RFC3339), + "content-type": "application/json", + }, + }) +} + +// eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able +// to tell a redelivery from a second event, and only the emitter can say which it is. +func eventID() (string, error) { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("cannot make an event id: %w", err) + } + return hex.EncodeToString(raw), nil +} + +// KeyModuleBuilt is what the builder announces when it has built something. The catalogue places +// it in the module graph; nothing else need care. +const KeyModuleBuilt = "module.builder.built"