Files
mesh-delivery/modules/mesh-delivery-builder/internal/build/candidates.go
T
jschoubben 84bcdb687a BL18: build modules found through their build seats' holders on the seat list
The builder finds a build module as the indexer's seed does: the build seat whose acting rule
`build` has an `on` matching the artifact's language or kind:image, and that seat's holder, read
from the seat list's definitions and the claims of the catalog half's manifests. Task.Module, every
lane key, the unpublished modules, the toolchain sources, the ends watched, the sayer and
builder_stuck (through `<seat>.steps`) derive from it; the floor's sources are the holders of the
seats git and mesh-catalog. buildrun.GoBuildModule, NodeBuildModule, LanguageOf,
buildstep.BuildModuleOf use, build.ImageBuildModule and StuckBuildModules are gone from the builder
(the checker keeps its own names in two small files until its part reads them from the list).

Builds: design 47 G §7, §428, §704, §204, A-82
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 00:04:34 +01:00

332 lines
13 KiB
Go

package build
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.<group>), and
// the builder builds each, once, as a key of its own (builds.<module>.<fingerprint>) that the checker's line reads. The
// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes
// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as
// the checker's fetch is, and builds the merge's module directory.
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
// commit the head is merged onto.
type via struct {
Ref, Base string
// Members are the group's package members the module is built against by their builds (design 47 G §373, §465).
Members []member
}
// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's
// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate.
type member struct{ Package, Module, Artifact, Fp string }
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
// carries. A value that is not a candidate is said and left.
func (b *Builder) Candidate(ctx context.Context, key string, value []byte) {
b.init()
c, err := meshcatalog.DecodeCandidatesByGroupValue(value)
if err != nil {
b.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
b.mu.Lock()
b.cands[key] = c
b.mu.Unlock()
b.considerCandidate(ctx, c)
}
// ForgetCandidate takes a candidate out (its key was deleted).
func (b *Builder) ForgetCandidate(key string) {
b.init()
b.mu.Lock()
delete(b.cands, key)
b.mu.Unlock()
}
func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) {
group := map[string]string{} // the fingerprint of each module the group's pull requests change
for _, p := range c.Pulls {
for name, cm := range p.Fps {
if cm.Fp2 != "" && cm.Manifest != "" {
group[name] = cm.Fp2
}
}
}
for _, p := range c.Pulls {
names := make([]string, 0, len(p.Fps))
for name := range p.Fps {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
cm := p.Fps[name]
if cm.Fp2 == "" || cm.Manifest == "" {
continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by
}
b.mu.Lock()
trunk, known := b.known[name]
b.mu.Unlock()
if !known || trunk.Path == "" {
b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.Fp2)
continue
}
key, err := builds.BuilderBuildKey(name, cm.Fp2)
if err != nil {
b.logf("candidates.%s: %v", c.Group, err)
continue
}
m := meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: p.Head, Fp2: cm.Fp2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, cm.Manifest)), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
if len(cm.Paths) > 0 {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Paths: cm.Paths}
}
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
base, err := b.trunkHead(ctx, uint64(c.Base), p.Repository)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
members, err := b.membersOf(ctx, name, cm, group)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members}
}
b.mu.Unlock()
b.consider(ctx, name, m)
}
}
}
// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the
// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed
// is left out, and the build then waits for it as the trunk's does.
func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]builds.ToolchainUsed {
plan, err, _ := b.planOf(ctx, manifest)
if err != nil {
return nil
}
out := map[string]builds.ToolchainUsed{}
for _, bm := range plan.Modules() {
if !plan.NeedsToolchain(bm) {
continue
}
language, _ := b.Seats.LanguageOf(bm)
if tc, none := buildstep.AgreedToolchain(b.Toolchains.Entries(language)); none == "" {
out[language] = builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
}
}
if len(out) == 0 {
return nil
}
return out
}
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
if b.Trunk == nil {
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
}
head, err := b.Trunk(ctx, rev, repository)
if err != nil {
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
}
if head == "" {
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
}
return head, nil
}
// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package
// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the
// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds,
// is no member: it is fetched as before.
func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) {
if b.Members == nil {
return nil, nil
}
var out []member
for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) {
if len(cm.Packages[pkg]) != 40 {
continue
}
module, artifact, found, err := b.Members(ctx, pkg)
if err != nil {
return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err)
}
if !found || module == name || group[module] == "" {
continue
}
out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]})
}
return out, nil
}
// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's
// 60-minute bound).
const MemberBound = 60 * time.Minute
// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77):
// Arm writes it for key, the dependent's build key; End deletes it.
type MemberBounds interface {
Arm(ctx context.Context, key, why string, bound time.Duration) error
End(ctx context.Context, key string)
}
// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the
// candidates are looked at again, which fails its step naming the member still not built.
func (b *Builder) MemberBoundEnded(ctx context.Context, key string) {
b.init()
b.mu.Lock()
waiting := b.memberWaits[key]
if waiting {
b.memberExpired[key] = true
}
b.mu.Unlock()
if waiting {
b.TrunkMoved(ctx)
}
}
// endMemberWait forgets key's wait and deletes its bound.
func (b *Builder) endMemberWait(ctx context.Context, key string) {
b.mu.Lock()
had := b.memberWaits[key]
delete(b.memberWaits, key)
delete(b.memberExpired, key)
b.mu.Unlock()
if had && b.MemberBounds != nil {
b.MemberBounds.End(ctx, key)
}
}
// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the
// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or
// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the
// builder holds no timer.
func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) {
b.mu.Lock()
v := b.via[key]
b.mu.Unlock()
if len(v.Members) == 0 {
return nil, "", false
}
defer func() {
if !waiting {
b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound
}
}()
var pending []string
for _, mem := range v.Members {
ref := mem.Module + "/" + mem.Artifact
mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp)
if err != nil {
return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false
}
raw, _, found, err := b.Claimer.Store.Get(ctx, mk)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false
}
if !found {
pending = append(pending, ref)
continue
}
cur, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false
}
switch cur.State {
case builds.BuildDone:
digest := ""
for _, a := range cur.Artifacts {
if a.Name == mem.Artifact {
digest = a.Digest
}
}
if digest == "" {
return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false
}
got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest})
case builds.BuildFailed:
return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false
default:
pending = append(pending, ref)
}
}
if len(pending) == 0 {
b.mu.Lock()
b.fetched[key] = got
b.mu.Unlock()
return got, "", false
}
b.mu.Lock()
said := b.memberWaits[key]
expired := b.memberExpired[key]
b.mu.Unlock()
if expired {
return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false
}
if !said {
line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String()
if b.MemberBounds != nil {
// The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written
// would never end, so the step fails at once, saying so.
if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil {
return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false
}
}
b.mu.Lock()
b.memberWaits[key] = true
b.mu.Unlock()
b.logf("%s", line)
if b.Log != nil {
b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"})
}
}
return nil, "", true
}
// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's
// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits.
func (b *Builder) memberEnded(ctx context.Context, key string) {
b.mu.Lock()
member := false
for _, v := range b.via {
for _, mem := range v.Members {
if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key {
member = true
}
}
}
b.mu.Unlock()
if member {
b.TrunkMoved(ctx)
}
}
// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none.
func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild {
b.mu.Lock()
defer b.mu.Unlock()
return b.fetched[key]
}