novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Builder: the dead candidate machinery goes

Browse Source 
Builder.Candidate, ForgetCandidate, considerCandidate, trunkHead, TrunkMoved, the cands and via
maps, the package-member wait that read via.Members (memberBuilds, memberEnded, MemberBoundEnded,
the member bound key) and the committed build output modules/mesh-delivery-builder/mesh-delivery-builder
are removed. Nothing fed Candidate in the process: the builder follows no candidate and learns a
run's builds from the build step ask. Their tests go with them.

Builds: design 47 G A-84 (the builder follows no candidate), §194, §348, §473
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
jschoubben
2026-11-12 01:04:23 +01:00
parent e1bf32b9fb
commit 62e56af786
10 changed files with 11 additions and 1008 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+1 -8
View File
@@ -251,8 +251,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
queueKV.Ready(again.bound(ctx, js, logger.Printf))
actedKV.Ready(again.bound(ctx, js, logger.Printf))
errs := make(chan error, 10)
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465)
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
b.Log = func(e builds.BuildLogEntry) {
@@ -462,12 +461,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
stepReads.expired(ctx, e)
return
}
if key, ok := versions.MemberBoundOf(e.Key()); ok {
if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member
b.MemberBoundEnded(ctx, key)
}
return
}
if node, ok := enginereads.BoundOf(e.Key()); ok {
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
reads.expiredCall(node)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go
-42
View File
@@ -1,42 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/versions"
)
// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of
// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded).
// No timer.
type memberBounds struct {
bounds func() jetstream.KeyValue
now func() time.Time
logf func(string, ...any)
}
// Arm is build.MemberBounds' Arm.
func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error {
kv := m.bounds()
if kv == nil {
return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
}
if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
return err
}
return nil
}
// End is build.MemberBounds' End.
func (m *memberBounds) End(ctx context.Context, key string) {
if kv := m.bounds(); kv != nil {
if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
m.logf("the member bound of %s is not deleted: %v", key, err)
}
}
}
modules/mesh-delivery-builder/internal/build/candidates.go
-299
View File
@@ -2,121 +2,11 @@ package build
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.<group>), and
// the builder builds each, once, as a key of its own (builds.<module>.<fingerprint>) that the checker's line reads. The
// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes
// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as
// the checker's fetch is, and builds the merge's module directory.
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
// commit the head is merged onto.
type via struct {
Ref, Base string
// Members are the group's package members the module is built against by their builds (design 47 G §373, §465).
Members []member
}
// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's
// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate.
type member struct{ Package, Module, Artifact, Fp string }
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
// carries. A value that is not a candidate is said and left.
func (b *Builder) Candidate(ctx context.Context, key string, value []byte) {
b.init()
c, err := meshcatalog.DecodeCandidatesByGroupValue(value)
if err != nil {
b.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
b.mu.Lock()
b.cands[key] = c
b.mu.Unlock()
b.considerCandidate(ctx, c)
}
// ForgetCandidate takes a candidate out (its key was deleted).
func (b *Builder) ForgetCandidate(key string) {
b.init()
b.mu.Lock()
delete(b.cands, key)
b.mu.Unlock()
}
func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) {
group := map[string]string{} // the fingerprint of each module the group's pull requests change
for _, p := range c.Pulls {
for name, cm := range p.Fps {
if cm.Fp2 != "" && cm.Manifest != "" {
group[name] = cm.Fp2
}
}
}
for _, p := range c.Pulls {
names := make([]string, 0, len(p.Fps))
for name := range p.Fps {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
cm := p.Fps[name]
if cm.Fp2 == "" || cm.Manifest == "" {
continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by
}
b.mu.Lock()
trunk, known := b.known[name]
b.mu.Unlock()
if !known || trunk.Path == "" {
b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.Fp2)
continue
}
key, err := builds.BuilderBuildKey(name, cm.Fp2)
if err != nil {
b.logf("candidates.%s: %v", c.Group, err)
continue
}
m := meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: p.Head, Fp2: cm.Fp2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, cm.Manifest)), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
if len(cm.Paths) > 0 {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Paths: cm.Paths}
}
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
base, err := b.trunkHead(ctx, uint64(c.Base), p.Repository)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
members, err := b.membersOf(ctx, name, cm, group)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members}
}
b.mu.Unlock()
b.consider(ctx, name, m)
}
}
}
// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the
// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed
// is left out, and the build then waits for it as the trunk's does.
@@ -140,192 +30,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]bui
}
return out
}
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
if b.Trunk == nil {
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
}
head, err := b.Trunk(ctx, rev, repository)
if err != nil {
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
}
if head == "" {
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
}
return head, nil
}
// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package
// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the
// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds,
// is no member: it is fetched as before.
func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) {
if b.Members == nil {
return nil, nil
}
var out []member
for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) {
if len(cm.Packages[pkg]) != 40 {
continue
}
module, artifact, found, err := b.Members(ctx, pkg)
if err != nil {
return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err)
}
if !found || module == name || group[module] == "" {
continue
}
out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]})
}
return out, nil
}
// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's
// 60-minute bound).
const MemberBound = 60 * time.Minute
// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77):
// Arm writes it for key, the dependent's build key; End deletes it.
type MemberBounds interface {
Arm(ctx context.Context, key, why string, bound time.Duration) error
End(ctx context.Context, key string)
}
// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the
// candidates are looked at again, which fails its step naming the member still not built.
func (b *Builder) MemberBoundEnded(ctx context.Context, key string) {
b.init()
b.mu.Lock()
waiting := b.memberWaits[key]
if waiting {
b.memberExpired[key] = true
}
b.mu.Unlock()
if waiting {
b.TrunkMoved(ctx)
}
}
// endMemberWait forgets key's wait and deletes its bound.
func (b *Builder) endMemberWait(ctx context.Context, key string) {
b.mu.Lock()
had := b.memberWaits[key]
delete(b.memberWaits, key)
delete(b.memberExpired, key)
b.mu.Unlock()
if had && b.MemberBounds != nil {
b.MemberBounds.End(ctx, key)
}
}
// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the
// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or
// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the
// builder holds no timer.
func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) {
b.mu.Lock()
v := b.via[key]
b.mu.Unlock()
if len(v.Members) == 0 {
return nil, "", false
}
defer func() {
if !waiting {
b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound
}
}()
var pending []string
for _, mem := range v.Members {
ref := mem.Module + "/" + mem.Artifact
mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp)
if err != nil {
return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false
}
raw, _, found, err := b.Claimer.Store.Get(ctx, mk)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false
}
if !found {
pending = append(pending, ref)
continue
}
cur, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false
}
switch cur.State {
case builds.BuildDone:
digest := ""
for _, a := range cur.Artifacts {
if a.Name == mem.Artifact {
digest = a.Digest
}
}
if digest == "" {
return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false
}
got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest})
case builds.BuildFailed:
return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false
default:
pending = append(pending, ref)
}
}
if len(pending) == 0 {
b.mu.Lock()
b.fetched[key] = got
b.mu.Unlock()
return got, "", false
}
b.mu.Lock()
said := b.memberWaits[key]
expired := b.memberExpired[key]
b.mu.Unlock()
if expired {
return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false
}
if !said {
line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String()
if b.MemberBounds != nil {
// The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written
// would never end, so the step fails at once, saying so.
if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil {
return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false
}
}
b.mu.Lock()
b.memberWaits[key] = true
b.mu.Unlock()
b.logf("%s", line)
if b.Log != nil {
b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"})
}
}
return nil, "", true
}
// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's
// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits.
func (b *Builder) memberEnded(ctx context.Context, key string) {
b.mu.Lock()
member := false
for _, v := range b.via {
for _, mem := range v.Members {
if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key {
member = true
}
}
}
b.mu.Unlock()
if member {
b.TrunkMoved(ctx)
}
}
// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none.
func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild {
b.mu.Lock()
defer b.mu.Unlock()
return b.fetched[key]
}
modules/mesh-delivery-builder/internal/build/candidates_test.go
-185
View File
@@ -1,185 +0,0 @@
package build
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1)
// is one the trunk does not carry.
func trunkModule(t *testing.T, r *rig) {
t.Helper()
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'),
Fp2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
r.b.Wait()
r.bm.mu.Lock()
r.bm.calls = nil
r.bm.mu.Unlock()
}
func candidateValue(t *testing.T, group string, number int, head string) []byte {
t.Helper()
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: int64(number), Head: head, Modules: []string{"m"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a')}}}}
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
return data
}
const (
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
// commit its fingerprint first appeared at.
trunkHead = "3333333333333333333333333333333333333333"
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
)
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c meshcatalog.CandidatesByGroupValue, raw []byte) {
t.Helper()
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
if err != nil {
t.Fatal(err)
}
var w struct {
Source struct {
Rev uint64 `json:"rev"`
Heads map[string]string `json:"heads"`
} `json:"source"`
Candidate json.RawMessage `json:"candidate"`
}
if err := json.Unmarshal(data, &w); err != nil {
t.Fatal(err)
}
if c, err = meshcatalog.DecodeCandidatesByGroupValue(w.Candidate); err != nil {
t.Fatal(err)
}
return w.Source.Rev, w.Source.Heads, c, w.Candidate
}
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
{
rev, heads, c, raw := indexerCandidate(t)
p := c.Pulls[0]
cm := p.Fps["avahi"]
ir := newRig(t, goBundle)
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
if at != rev {
return "", fmt.Errorf("source revision %d is not heard", at)
}
return heads[repository], nil
}
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: valueCommit, Fp2: "src2:" + hex64('e'),
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: CatalogToolchains(agreedToolchains)})
if err != nil {
t.Fatal(err)
}
ir.b.Module(context.Background(), "avahi", value)
ir.b.Wait()
ir.bm.mu.Lock()
ir.bm.calls = nil
ir.bm.mu.Unlock()
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
ir.b.Wait()
fetches := ir.bm.callsOf("build_go_fetch")
if len(fetches) != 1 {
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head {
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
}
}
r := newRig(t, goBundle)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildDone || len(v.Artifacts) != 1 {
t.Fatalf("build = %+v, want the candidate's fingerprint built", v)
}
fetches := r.bm.callsOf("build_go_fetch")
if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4)
t.Fatalf("%d fetches, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
}
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
t.Errorf("build args = %+v", b)
}
// Two heads carrying one fingerprint are one build.
r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY))
r.b.Wait()
if n := len(r.bm.callsOf("build_go_fetch")); n != 1 {
t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n)
}
}
func TestACandidateWhoseModuleHasNoTrunkRecordIsNotBuilt(t *testing.T) {
r := newRig(t, goBundle)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if _, ok := r.st.vals[r.key]; ok {
t.Error("a build key was written for a module the builder knows no directory of")
}
}
func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { // the trunk's source wins
r := newRig(t, goBundle)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the trunk now carries fp1
r.b.Wait()
for _, c := range r.bm.callsOf("build_go_fetch") {
if f := c.body.(map[string]any); f["ref"] != nil || f["base"] != nil {
t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f)
}
}
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Errorf("build = %+v", v)
}
}
func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t *testing.T) { // review D4
for name, set := range map[string]func(*rig){
"conflict": func(r *rig) { r.bm.conflicts = "go.mod" },
"no merge": func(r *rig) { r.bm.noMerge = true },
} {
r := newRig(t, goBundle)
trunkModule(t, r)
set(r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 {
t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build")))
}
}
}
modules/mesh-delivery-builder/internal/build/members_test.go
-166
View File
@@ -1,166 +0,0 @@
package build
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const memberPackage = "example.org/p"
var fpP = "src2:" + hex64('2')
// groupValue is a group's candidate: the module m, built against the head of the package member p, and p.
func groupValue(t *testing.T) []byte {
t.Helper()
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{
"m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}},
"p": {Fp2: fpP, Manifest: mdigest('b')}}}}
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}})
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
return data
}
// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known.
func memberRig(t *testing.T, known bool) *rig {
t.Helper()
r := newRig(t, goBundle)
lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"}
r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}}
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
if pkg == memberPackage {
return "p", "lib", true, nil
}
return "", "", false, nil
}
trunkModule(t, r)
if known {
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'),
Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "p", data)
r.b.Wait()
}
return r
}
func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103
t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) {
r := memberRig(t, true)
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("m = %+v, want built", v)
}
var found bool
for _, c := range r.bm.callsOf("build_go_fetch") {
f := c.body.(map[string]any)
bs, _ := f["builds"].([]any)
if len(bs) == 0 {
continue
}
found = true
want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest}
if len(bs) != 1 || !jsonEqual(bs[0], want) {
t.Errorf("builds = %v, want %v", bs, want)
}
pk, _ := f["packages"].(map[string]any)
if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" {
t.Errorf("packages = %v, want the member out of it and the other package kept", pk)
}
}
if !found {
t.Error("no fetch named the member's build")
}
})
t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) {
r := memberRig(t, false)
var mu sync.Mutex
bounds := &fakeMemberBounds{}
r.b.MemberBounds = bounds
var said []builds.BuildLogEntry
r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() }
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Fatal("the dependent was claimed or built while its package member is not built")
}
mu.Lock()
if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") {
t.Errorf("said = %+v, want one wait-held naming p/lib", said)
}
mu.Unlock()
if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() {
t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got)
}
r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry
r.b.Wait()
if got := bounds.ended(); len(got) != 1 || got[0] != r.key {
t.Errorf("bounds ended = %v, want the wait's bound deleted", got)
}
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Errorf("m = %+v, want failed naming the member and nothing built", v)
}
})
t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) {
r := memberRig(t, false)
mk, _ := builds.BuilderBuildKey("p", fpP)
data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"})
if err != nil {
t.Fatal(err)
}
r.st.put(mk, data)
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Errorf("m = %+v, want failed naming the member and its error", v)
}
})
}
// fakeMemberBounds keeps the bounds a builder arms and ends.
type fakeMemberBounds struct {
mu sync.Mutex
arms, ends_ []string
}
func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error {
f.mu.Lock()
defer f.mu.Unlock()
f.arms = append(f.arms, key+" "+bound.String())
return nil
}
func (f *fakeMemberBounds) End(_ context.Context, key string) {
f.mu.Lock()
defer f.mu.Unlock()
f.ends_ = append(f.ends_, key)
}
func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms }
func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ }
func jsonEqual(a, b any) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y)
}
modules/mesh-delivery-builder/internal/build/r4e_test.go
+3 -121
View File
@@ -2,45 +2,19 @@ package build
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait
// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4).
func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3
r := newRig(t, goBundle, archiveA)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildDone {
t.Fatalf("build = %+v, want done", v)
}
if ok, missing := v.Referenced(); !ok {
t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts)
}
rev := r.st.revs[r.key]
r.bm.mu.Lock()
r.bm.calls = nil
r.bm.mu.Unlock()
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint
r.b.Wait()
if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev {
t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev)
}
}
func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2
r := newRig(t, goBundle)
r.bm.noRef["ace/tool"] = true
@@ -63,7 +37,7 @@ func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1
}
}
func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key
func TestATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a)
r := newRig(t, goBundle)
r.consider(t, "modules/m")
for _, c := range r.bm.callsOf("build_go_build") {
@@ -71,15 +45,6 @@ func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's
t.Errorf("trunk build args = %+v, want publish", c.body)
}
}
c := newRig(t, goBundle)
trunkModule(t, c)
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
c.b.Wait()
for _, call := range c.bm.callsOf("build_go_build") {
if _, ok := call.body.(map[string]any)["publish"]; ok {
t.Errorf("candidate build args = %+v, want no publish", call.body)
}
}
}
// putBuild writes a builder key's value as the store would hold it.
@@ -164,25 +129,10 @@ func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4
var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"}
func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
c := newRig(t, tsPackage)
c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, c)
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
c.b.Wait()
v := valueOf(t, c.st, c.key)
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" {
t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v)
}
for _, call := range c.bm.callsOf("build_node_build") {
if _, ok := call.body.(map[string]any)["publish"]; ok {
t.Errorf("a candidate's package build was asked to publish: %+v", call.body)
}
}
func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublished(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
v = r.consider(t, "modules/m")
v := r.consider(t, "modules/m")
if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" {
t.Fatalf("trunk build = %+v, want done naming the published version", v)
}
@@ -207,50 +157,6 @@ func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B
}
}
func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6
r := newRig(t, tsPackage, tsBundle)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("candidate build = %+v", v)
}
rev := r.st.revs[r.key]
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint
r.b.Wait()
v := valueOf(t, r.st, r.key)
if r.st.revs[r.key] <= rev {
t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev)
}
if ok, missing := v.Referenced(); !ok {
t.Fatalf("build = %+v, missing references %v", v, missing)
}
rev = r.st.revs[r.key]
r.b.tried = map[string]bool{}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if r.st.revs[r.key] != rev {
t.Error("a published package was built again: once only")
}
}
func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
for _, n := range []string{"ace", "novox"} {
r.bm.fails[n+"/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) {
t.Errorf("build = %+v, want failed opening %q", v, builds.VersionTaken)
}
}
// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref.
func sdkModule(t *testing.T, r *rig, ref string) []byte {
t.Helper()
@@ -365,30 +271,6 @@ func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9:
}
}
func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}}
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
r.b.Candidate(context.Background(), "feat%2Fx", data)
r.b.Wait()
f := r.bm.callsOf("build_node_fetch")
if len(f) == 0 {
t.Fatal("no fetch")
}
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
}
}
// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
// answered a digest alone (a build module from before R4e).
func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
modules/mesh-delivery-builder/internal/build/run.go
+7 -73
View File
@@ -77,14 +77,9 @@ type Builder struct {
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
Stored func(ctx context.Context, references []string) (gone []string, err error)
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
// §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a
// member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none.
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
MemberBounds MemberBounds
// §373, §465): the part a resolved package names in a build's dependencies. Nil names none.
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
@@ -132,8 +127,6 @@ type Builder struct {
again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built
plans map[string]planned // by manifest digest
stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go)
cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go)
via map[string]via // by build key: a candidate's fetch (candidates.go)
// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
outOfStep map[string][]string
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
@@ -141,10 +134,6 @@ type Builder struct {
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
// waiting for one is built once it is here (ADR 0500 clause 6).
published map[string]bool
// memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired
// those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go).
memberWaits, memberExpired map[string]bool
fetched map[string][]sdkstep.FetchBuild
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool
@@ -181,13 +170,12 @@ func (b *Builder) init() {
limit = 256
}
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{}
b.trunkFP = map[string]bool{}
b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{}
b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{}
b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{}
b.builds = map[string]context.CancelCauseFunc{}
})
}
@@ -265,7 +253,6 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
b.trunkFP[bk] = true
b.origin[bk] = o
b.edges[bk] = e
delete(b.via, bk)
}
b.mu.Unlock()
if superseded != nil {
@@ -472,23 +459,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
for _, k := range sortedKeys(known) {
b.consider(ctx, k, known[k])
}
b.TrunkMoved(ctx)
}
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
// revision it was previewed onto had no trunk head to be merged onto.
func (b *Builder) TrunkMoved(ctx context.Context) {
b.init()
b.mu.Lock()
cands := make([]meshcatalog.CandidatesByGroupValue, 0, len(b.cands))
for _, c := range b.cands {
cands = append(cands, c)
}
b.mu.Unlock()
sort.Slice(cands, func(i, j int) bool { return cands[i].Group < cands[j].Group })
for _, c := range cands {
b.considerCandidate(ctx, c)
}
}
func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string {
@@ -630,13 +600,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
return
}
}
memberFail := ""
if !trunk && planErr == nil && !nothing {
var waiting bool
if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting {
return // a package member of the group is not built (design 47 G §373): its end looks at this key again
}
}
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
if reserving && b.waitFill(module, m) {
@@ -654,7 +617,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
return
}
b.handClaimed(key, cl.Rev)
b.logBuild(LogStarted(module, m, b.refOf(module, m.Fp2), b.Node, b.now()))
b.logBuild(LogStarted(module, m, "", b.Node, b.now()))
if cl.Unasked != nil {
b.logf("%v", cl.Unasked)
}
@@ -686,8 +649,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
// fails with why, and is taken again at the next look.
result = failed("%s", numbering.Refused)
} else if memberFail != "" && cl.Build.State != builds.BuildDone {
result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373)
} else if nothing {
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
@@ -734,8 +695,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
if b.Ended != nil {
b.Ended(ctx, cl.Key)
}
b.memberEnded(ctx, cl.Key)
ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result)
ended := LogEnded(module, m, "", b.Node, b.now(), result)
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
b.logBuild(ended)
if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
@@ -1074,15 +1034,6 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
// key taken over at start waits for the ask the key names and puts none.
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) {
req := sdkstep.Fetch{Repository: repository, Commit: m.Commit}
if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
b.mu.Lock()
v, candidate := b.via[key]
b.mu.Unlock()
if candidate {
req.Ref, req.Base = v.Ref, v.Base
req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465)
}
}
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
@@ -1092,17 +1043,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
}
sort.Strings(req.Paths)
}
candidate := req.Base != ""
seat, _ := b.Seats.OfModule(l.module)
language := seat.AskLanguage()
kind := sdkstep.AskDelivery
if candidate {
kind = sdkstep.AskCheck
}
job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain),
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
Publish: !candidate}
Publish: true}
for _, t := range l.tasks {
a := sdkstep.AskArtifact{Artifact: t.Artifact}
if t.Kind == catalogue.ArtifactUpstream {
@@ -1197,7 +1142,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
}
l.digests[t.Artifact] = a.Digest
l.resolved[t.Artifact] = a.Resolved
if a.Reference != "" && (stored(t.Kind) || !candidate) {
if a.Reference != "" {
l.references[t.Artifact] = a.Reference
}
if a.Reference != "" && b.Answered != nil {
@@ -1236,17 +1181,6 @@ func (b *Builder) logBuild(e builds.BuildLogEntry) {
b.Log(e)
}
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
func (b *Builder) refOf(module, fp string) string {
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return ""
}
b.mu.Lock()
defer b.mu.Unlock()
return b.via[key].Ref
}
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
// that asked it and the time.
modules/mesh-delivery-builder/internal/build/run_test.go
-101
View File
@@ -187,7 +187,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
r.b = &Builder{
Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
}
var err error
if r.key, err = builds.BuilderBuildKey("m", fp1); err != nil {
@@ -437,31 +436,6 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
}
}
func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision
r := newRig(t, imageArt)
r.puller = "ace"
trunkModule(t, r)
// Design 47 G §373: a candidate's build is numbered under no tag, so it numbers nothing.
nums := &fakeNumbers{}
r.b.Numbers = nums
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if nums.numbered != 0 {
t.Errorf("a candidate's build was numbered %d times, want none", nums.numbered)
}
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" {
t.Fatalf("build = %+v, want done with no reference", v)
}
calls := r.bm.callsOf("build_image_build")
if len(calls) != 1 {
t.Fatalf("%d image builds, want 1", len(calls))
}
if _, has := calls[0].body.(map[string]any)["publish"]; has {
t.Errorf("a candidate's image call carries publish: %+v", calls[0].body)
}
}
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim
r := newRig(t, imageArt)
r.puller = "ace"
@@ -793,81 +767,6 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/
}
}
func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5
r := newRig(t, imageArt)
r.puller = "ace"
trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildDone || v.Artifacts[0].Reference != "" {
t.Fatalf("candidate build = %+v, want done with no reference", v)
}
if n := len(r.bm.callsOf("build_image_build")); n != 1 {
t.Fatalf("%d image builds after the candidate, want 1", n)
}
// The pull request is merged: the trunk's record names the candidate's fingerprint.
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
v = valueOf(t, r.st, r.key)
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" {
t.Fatalf("trunk build = %+v, want done with a reference", v)
}
calls := r.bm.callsOf("build_image_build")
published := 0
for _, c := range calls {
if c.body.(map[string]any)["publish"] == true {
published++
}
}
if len(calls) != 2 || published != 1 {
t.Errorf("%d image builds, %d with publish, want 2 and exactly 1", len(calls), published)
}
// Written once more only: looking again builds nothing.
rev := r.st.revs[r.key]
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_image_build")) != 2 {
t.Error("a published key was built again")
}
}
func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
r := newRig(t, imageArt)
r.puller = "ace"
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("candidate build = %+v", v)
}
r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused"
// Issue 1748: a run of the ask that ended on a transport fault is said in the log and on the key, and is no end.
inner, log := r.asks.Answer, &logged{}
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
r.asks.Write(buildstep.Step{ID: a.ID, Node: "g14", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Reason: "lookup registry: i/o timeout"})
return inner(a)
}
r.b.Logf = log.logf
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint
r.b.Wait()
r.asks.Answer, r.b.Logf = inner, t.Logf
if !log.has("run 1 of", "on g14 ended on a transport fault", "lookup registry: i/o timeout") {
t.Errorf("a run given again was not said: %v", log.lines)
}
v := valueOf(t, r.st, r.key)
if v.State != builds.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") {
t.Fatalf("build = %+v, want failed naming the publish and its reason", v)
}
// The failed key is retaken as any failed key is, and ends published once the registry answers.
delete(r.bm.fails, "ace/build_image_build")
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Artifacts[0].Reference == "" {
t.Errorf("build = %+v, want done with a reference", v)
}
}
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
// drop from their stream (Dropper).
type goneAsks struct {
modules/mesh-delivery-builder/internal/build/step.go
-5
View File
@@ -73,11 +73,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{}
}
b.mu.Unlock()
b.consider(ctx, entry.Module, m)
select {
case log := <-ended:
modules/mesh-delivery-builder/internal/versions/publish.go
-8
View File
@@ -38,16 +38,8 @@ const (
oldFormBoundPrefix = "old-form."
unputBoundPrefix = "unput."
stepsBoundPrefix = "steps."
memberBoundPrefix = "member."
)
// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77):
// `member.<build key>`. Its end while the member is not built fails the dependent's step naming the member.
func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey }
// MemberBoundOf is the build key a member wait's bound names, and whether key is one.
func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) }
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
// novox/hq ADR 0668 §3): `unput.<build key>`. Its end while the key still stands so, with no look holding it, is the
// finding call-unanswered about the key.