Builder: the dead candidate machinery goes
Builder.Candidate, ForgetCandidate, considerCandidate, trunkHead, TrunkMoved, the cands and via maps, the package-member wait that read via.Members (memberBuilds, memberEnded, MemberBoundEnded, the member bound key) and the committed build output modules/mesh-delivery-builder/mesh-delivery-builder are removed. Nothing fed Candidate in the process: the builder follows no candidate and learns a run's builds from the build step ask. Their tests go with them. Builds: design 47 G A-84 (the builder follows no candidate), §194, §348, §473 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -251,8 +251,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
queueKV.Ready(again.bound(ctx, js, logger.Printf))
|
queueKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||||
actedKV.Ready(again.bound(ctx, js, logger.Printf))
|
actedKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||||
errs := make(chan error, 10)
|
errs := make(chan error, 10)
|
||||||
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
|
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
|
||||||
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465)
|
|
||||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||||
b.Log = func(e builds.BuildLogEntry) {
|
b.Log = func(e builds.BuildLogEntry) {
|
||||||
@@ -462,12 +461,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
stepReads.expired(ctx, e)
|
stepReads.expired(ctx, e)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if key, ok := versions.MemberBoundOf(e.Key()); ok {
|
|
||||||
if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member
|
|
||||||
b.MemberBoundEnded(ctx, key)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if node, ok := enginereads.BoundOf(e.Key()); ok {
|
if node, ok := enginereads.BoundOf(e.Key()); ok {
|
||||||
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
|
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
|
||||||
reads.expiredCall(node)
|
reads.expiredCall(node)
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"mesh-delivery-builder/internal/versions"
|
|
||||||
)
|
|
||||||
|
|
||||||
// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of
|
|
||||||
// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded).
|
|
||||||
// No timer.
|
|
||||||
type memberBounds struct {
|
|
||||||
bounds func() jetstream.KeyValue
|
|
||||||
now func() time.Time
|
|
||||||
logf func(string, ...any)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Arm is build.MemberBounds' Arm.
|
|
||||||
func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error {
|
|
||||||
kv := m.bounds()
|
|
||||||
if kv == nil {
|
|
||||||
return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
|
|
||||||
}
|
|
||||||
if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// End is build.MemberBounds' End.
|
|
||||||
func (m *memberBounds) End(ctx context.Context, key string) {
|
|
||||||
if kv := m.bounds(); kv != nil {
|
|
||||||
if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
||||||
m.logf("the member bound of %s is not deleted: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,121 +2,11 @@ package build
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"maps"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||||
)
|
)
|
||||||
|
|
||||||
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
|
|
||||||
// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.<group>), and
|
|
||||||
// the builder builds each, once, as a key of its own (builds.<module>.<fingerprint>) that the checker's line reads. The
|
|
||||||
// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes
|
|
||||||
// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as
|
|
||||||
// the checker's fetch is, and builds the merge's module directory.
|
|
||||||
|
|
||||||
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
|
|
||||||
// commit the head is merged onto.
|
|
||||||
type via struct {
|
|
||||||
Ref, Base string
|
|
||||||
// Members are the group's package members the module is built against by their builds (design 47 G §373, §465).
|
|
||||||
Members []member
|
|
||||||
}
|
|
||||||
|
|
||||||
// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's
|
|
||||||
// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate.
|
|
||||||
type member struct{ Package, Module, Artifact, Fp string }
|
|
||||||
|
|
||||||
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
|
|
||||||
// carries. A value that is not a candidate is said and left.
|
|
||||||
func (b *Builder) Candidate(ctx context.Context, key string, value []byte) {
|
|
||||||
b.init()
|
|
||||||
c, err := meshcatalog.DecodeCandidatesByGroupValue(value)
|
|
||||||
if err != nil {
|
|
||||||
b.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
b.cands[key] = c
|
|
||||||
b.mu.Unlock()
|
|
||||||
b.considerCandidate(ctx, c)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ForgetCandidate takes a candidate out (its key was deleted).
|
|
||||||
func (b *Builder) ForgetCandidate(key string) {
|
|
||||||
b.init()
|
|
||||||
b.mu.Lock()
|
|
||||||
delete(b.cands, key)
|
|
||||||
b.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) {
|
|
||||||
group := map[string]string{} // the fingerprint of each module the group's pull requests change
|
|
||||||
for _, p := range c.Pulls {
|
|
||||||
for name, cm := range p.Fps {
|
|
||||||
if cm.Fp2 != "" && cm.Manifest != "" {
|
|
||||||
group[name] = cm.Fp2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, p := range c.Pulls {
|
|
||||||
names := make([]string, 0, len(p.Fps))
|
|
||||||
for name := range p.Fps {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
for _, name := range names {
|
|
||||||
cm := p.Fps[name]
|
|
||||||
if cm.Fp2 == "" || cm.Manifest == "" {
|
|
||||||
continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
trunk, known := b.known[name]
|
|
||||||
b.mu.Unlock()
|
|
||||||
if !known || trunk.Path == "" {
|
|
||||||
b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.Fp2)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key, err := builds.BuilderBuildKey(name, cm.Fp2)
|
|
||||||
if err != nil {
|
|
||||||
b.logf("candidates.%s: %v", c.Group, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m := meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: p.Head, Fp2: cm.Fp2, Manifest: cm.Manifest, Path: trunk.Path,
|
|
||||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, cm.Manifest)), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
|
|
||||||
if len(cm.Paths) > 0 {
|
|
||||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
|
||||||
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Paths: cm.Paths}
|
|
||||||
}
|
|
||||||
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
|
||||||
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
|
||||||
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
|
||||||
base, err := b.trunkHead(ctx, uint64(c.Base), p.Repository)
|
|
||||||
if err != nil {
|
|
||||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
members, err := b.membersOf(ctx, name, cm, group)
|
|
||||||
if err != nil {
|
|
||||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
|
||||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members}
|
|
||||||
}
|
|
||||||
b.mu.Unlock()
|
|
||||||
b.consider(ctx, name, m)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the
|
// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the
|
||||||
// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed
|
// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed
|
||||||
// is left out, and the build then waits for it as the trunk's does.
|
// is left out, and the build then waits for it as the trunk's does.
|
||||||
@@ -140,192 +30,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]bui
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
|
||||||
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
|
||||||
if b.Trunk == nil {
|
|
||||||
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
|
||||||
}
|
|
||||||
head, err := b.Trunk(ctx, rev, repository)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
|
||||||
}
|
|
||||||
if head == "" {
|
|
||||||
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
|
||||||
}
|
|
||||||
return head, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package
|
|
||||||
// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the
|
|
||||||
// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds,
|
|
||||||
// is no member: it is fetched as before.
|
|
||||||
func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) {
|
|
||||||
if b.Members == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var out []member
|
|
||||||
for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) {
|
|
||||||
if len(cm.Packages[pkg]) != 40 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
module, artifact, found, err := b.Members(ctx, pkg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err)
|
|
||||||
}
|
|
||||||
if !found || module == name || group[module] == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's
|
|
||||||
// 60-minute bound).
|
|
||||||
const MemberBound = 60 * time.Minute
|
|
||||||
|
|
||||||
// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77):
|
|
||||||
// Arm writes it for key, the dependent's build key; End deletes it.
|
|
||||||
type MemberBounds interface {
|
|
||||||
Arm(ctx context.Context, key, why string, bound time.Duration) error
|
|
||||||
End(ctx context.Context, key string)
|
|
||||||
}
|
|
||||||
|
|
||||||
// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the
|
|
||||||
// candidates are looked at again, which fails its step naming the member still not built.
|
|
||||||
func (b *Builder) MemberBoundEnded(ctx context.Context, key string) {
|
|
||||||
b.init()
|
|
||||||
b.mu.Lock()
|
|
||||||
waiting := b.memberWaits[key]
|
|
||||||
if waiting {
|
|
||||||
b.memberExpired[key] = true
|
|
||||||
}
|
|
||||||
b.mu.Unlock()
|
|
||||||
if waiting {
|
|
||||||
b.TrunkMoved(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// endMemberWait forgets key's wait and deletes its bound.
|
|
||||||
func (b *Builder) endMemberWait(ctx context.Context, key string) {
|
|
||||||
b.mu.Lock()
|
|
||||||
had := b.memberWaits[key]
|
|
||||||
delete(b.memberWaits, key)
|
|
||||||
delete(b.memberExpired, key)
|
|
||||||
b.mu.Unlock()
|
|
||||||
if had && b.MemberBounds != nil {
|
|
||||||
b.MemberBounds.End(ctx, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the
|
|
||||||
// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or
|
|
||||||
// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the
|
|
||||||
// builder holds no timer.
|
|
||||||
func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) {
|
|
||||||
b.mu.Lock()
|
|
||||||
v := b.via[key]
|
|
||||||
b.mu.Unlock()
|
|
||||||
if len(v.Members) == 0 {
|
|
||||||
return nil, "", false
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if !waiting {
|
|
||||||
b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
var pending []string
|
|
||||||
for _, mem := range v.Members {
|
|
||||||
ref := mem.Module + "/" + mem.Artifact
|
|
||||||
mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false
|
|
||||||
}
|
|
||||||
raw, _, found, err := b.Claimer.Store.Get(ctx, mk)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
pending = append(pending, ref)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
cur, err := builds.DecodeBuilderBuild(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false
|
|
||||||
}
|
|
||||||
switch cur.State {
|
|
||||||
case builds.BuildDone:
|
|
||||||
digest := ""
|
|
||||||
for _, a := range cur.Artifacts {
|
|
||||||
if a.Name == mem.Artifact {
|
|
||||||
digest = a.Digest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if digest == "" {
|
|
||||||
return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false
|
|
||||||
}
|
|
||||||
got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest})
|
|
||||||
case builds.BuildFailed:
|
|
||||||
return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false
|
|
||||||
default:
|
|
||||||
pending = append(pending, ref)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(pending) == 0 {
|
|
||||||
b.mu.Lock()
|
|
||||||
b.fetched[key] = got
|
|
||||||
b.mu.Unlock()
|
|
||||||
return got, "", false
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
said := b.memberWaits[key]
|
|
||||||
expired := b.memberExpired[key]
|
|
||||||
b.mu.Unlock()
|
|
||||||
if expired {
|
|
||||||
return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false
|
|
||||||
}
|
|
||||||
if !said {
|
|
||||||
line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String()
|
|
||||||
if b.MemberBounds != nil {
|
|
||||||
// The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written
|
|
||||||
// would never end, so the step fails at once, saying so.
|
|
||||||
if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil {
|
|
||||||
return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
b.memberWaits[key] = true
|
|
||||||
b.mu.Unlock()
|
|
||||||
b.logf("%s", line)
|
|
||||||
if b.Log != nil {
|
|
||||||
b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, "", true
|
|
||||||
}
|
|
||||||
|
|
||||||
// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's
|
|
||||||
// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits.
|
|
||||||
func (b *Builder) memberEnded(ctx context.Context, key string) {
|
|
||||||
b.mu.Lock()
|
|
||||||
member := false
|
|
||||||
for _, v := range b.via {
|
|
||||||
for _, mem := range v.Members {
|
|
||||||
if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key {
|
|
||||||
member = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.mu.Unlock()
|
|
||||||
if member {
|
|
||||||
b.TrunkMoved(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none.
|
|
||||||
func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
return b.fetched[key]
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,185 +0,0 @@
|
|||||||
package build
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
||||||
)
|
|
||||||
|
|
||||||
// trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1)
|
|
||||||
// is one the trunk does not carry.
|
|
||||||
func trunkModule(t *testing.T, r *rig) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'),
|
|
||||||
Fp2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
r.b.Module(context.Background(), "m", data)
|
|
||||||
r.b.Wait()
|
|
||||||
r.bm.mu.Lock()
|
|
||||||
r.bm.calls = nil
|
|
||||||
r.bm.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func candidateValue(t *testing.T, group string, number int, head string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: int64(number), Head: head, Modules: []string{"m"},
|
|
||||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a')}}}}
|
|
||||||
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
|
||||||
// commit its fingerprint first appeared at.
|
|
||||||
trunkHead = "3333333333333333333333333333333333333333"
|
|
||||||
headX = "1111111111111111111111111111111111111111"
|
|
||||||
headY = "2222222222222222222222222222222222222222"
|
|
||||||
)
|
|
||||||
|
|
||||||
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
|
||||||
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
|
||||||
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c meshcatalog.CandidatesByGroupValue, raw []byte) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var w struct {
|
|
||||||
Source struct {
|
|
||||||
Rev uint64 `json:"rev"`
|
|
||||||
Heads map[string]string `json:"heads"`
|
|
||||||
} `json:"source"`
|
|
||||||
Candidate json.RawMessage `json:"candidate"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(data, &w); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if c, err = meshcatalog.DecodeCandidatesByGroupValue(w.Candidate); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
|
||||||
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
|
||||||
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
|
||||||
{
|
|
||||||
rev, heads, c, raw := indexerCandidate(t)
|
|
||||||
p := c.Pulls[0]
|
|
||||||
cm := p.Fps["avahi"]
|
|
||||||
ir := newRig(t, goBundle)
|
|
||||||
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
|
||||||
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
|
||||||
if at != rev {
|
|
||||||
return "", fmt.Errorf("source revision %d is not heard", at)
|
|
||||||
}
|
|
||||||
return heads[repository], nil
|
|
||||||
}
|
|
||||||
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
|
||||||
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: valueCommit, Fp2: "src2:" + hex64('e'),
|
|
||||||
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: CatalogToolchains(agreedToolchains)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ir.b.Module(context.Background(), "avahi", value)
|
|
||||||
ir.b.Wait()
|
|
||||||
ir.bm.mu.Lock()
|
|
||||||
ir.bm.calls = nil
|
|
||||||
ir.bm.mu.Unlock()
|
|
||||||
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
|
||||||
ir.b.Wait()
|
|
||||||
fetches := ir.bm.callsOf("build_go_fetch")
|
|
||||||
if len(fetches) != 1 {
|
|
||||||
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
|
||||||
}
|
|
||||||
f := fetches[0].body.(map[string]any)
|
|
||||||
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head {
|
|
||||||
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
trunkModule(t, r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildDone || len(v.Artifacts) != 1 {
|
|
||||||
t.Fatalf("build = %+v, want the candidate's fingerprint built", v)
|
|
||||||
}
|
|
||||||
fetches := r.bm.callsOf("build_go_fetch")
|
|
||||||
if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4)
|
|
||||||
t.Fatalf("%d fetches, want one", len(fetches))
|
|
||||||
}
|
|
||||||
f := fetches[0].body.(map[string]any)
|
|
||||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
|
||||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
|
||||||
}
|
|
||||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
|
||||||
t.Errorf("build args = %+v", b)
|
|
||||||
}
|
|
||||||
// Two heads carrying one fingerprint are one build.
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY))
|
|
||||||
r.b.Wait()
|
|
||||||
if n := len(r.bm.callsOf("build_go_fetch")); n != 1 {
|
|
||||||
t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACandidateWhoseModuleHasNoTrunkRecordIsNotBuilt(t *testing.T) {
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
if _, ok := r.st.vals[r.key]; ok {
|
|
||||||
t.Error("a build key was written for a module the builder knows no directory of")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { // the trunk's source wins
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the trunk now carries fp1
|
|
||||||
r.b.Wait()
|
|
||||||
for _, c := range r.bm.callsOf("build_go_fetch") {
|
|
||||||
if f := c.body.(map[string]any); f["ref"] != nil || f["base"] != nil {
|
|
||||||
t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
|
||||||
t.Errorf("build = %+v", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t *testing.T) { // review D4
|
|
||||||
for name, set := range map[string]func(*rig){
|
|
||||||
"conflict": func(r *rig) { r.bm.conflicts = "go.mod" },
|
|
||||||
"no merge": func(r *rig) { r.bm.noMerge = true },
|
|
||||||
} {
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
trunkModule(t, r)
|
|
||||||
set(r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 {
|
|
||||||
t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build")))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,166 +0,0 @@
|
|||||||
package build
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
||||||
)
|
|
||||||
|
|
||||||
const memberPackage = "example.org/p"
|
|
||||||
|
|
||||||
var fpP = "src2:" + hex64('2')
|
|
||||||
|
|
||||||
// groupValue is a group's candidate: the module m, built against the head of the package member p, and p.
|
|
||||||
func groupValue(t *testing.T) []byte {
|
|
||||||
t.Helper()
|
|
||||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"},
|
|
||||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{
|
|
||||||
"m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}},
|
|
||||||
"p": {Fp2: fpP, Manifest: mdigest('b')}}}}
|
|
||||||
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known.
|
|
||||||
func memberRig(t *testing.T, known bool) *rig {
|
|
||||||
t.Helper()
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"}
|
|
||||||
r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}}
|
|
||||||
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
|
|
||||||
if pkg == memberPackage {
|
|
||||||
return "p", "lib", true, nil
|
|
||||||
}
|
|
||||||
return "", "", false, nil
|
|
||||||
}
|
|
||||||
trunkModule(t, r)
|
|
||||||
if known {
|
|
||||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'),
|
|
||||||
Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
r.b.Module(context.Background(), "p", data)
|
|
||||||
r.b.Wait()
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103
|
|
||||||
t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) {
|
|
||||||
r := memberRig(t, true)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
|
||||||
t.Fatalf("m = %+v, want built", v)
|
|
||||||
}
|
|
||||||
var found bool
|
|
||||||
for _, c := range r.bm.callsOf("build_go_fetch") {
|
|
||||||
f := c.body.(map[string]any)
|
|
||||||
bs, _ := f["builds"].([]any)
|
|
||||||
if len(bs) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
found = true
|
|
||||||
want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest}
|
|
||||||
if len(bs) != 1 || !jsonEqual(bs[0], want) {
|
|
||||||
t.Errorf("builds = %v, want %v", bs, want)
|
|
||||||
}
|
|
||||||
pk, _ := f["packages"].(map[string]any)
|
|
||||||
if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" {
|
|
||||||
t.Errorf("packages = %v, want the member out of it and the other package kept", pk)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Error("no fetch named the member's build")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) {
|
|
||||||
r := memberRig(t, false)
|
|
||||||
var mu sync.Mutex
|
|
||||||
bounds := &fakeMemberBounds{}
|
|
||||||
r.b.MemberBounds = bounds
|
|
||||||
var said []builds.BuildLogEntry
|
|
||||||
r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
|
||||||
r.b.Wait()
|
|
||||||
if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
|
||||||
t.Fatal("the dependent was claimed or built while its package member is not built")
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") {
|
|
||||||
t.Errorf("said = %+v, want one wait-held naming p/lib", said)
|
|
||||||
}
|
|
||||||
mu.Unlock()
|
|
||||||
if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() {
|
|
||||||
t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got)
|
|
||||||
}
|
|
||||||
r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry
|
|
||||||
r.b.Wait()
|
|
||||||
if got := bounds.ended(); len(got) != 1 || got[0] != r.key {
|
|
||||||
t.Errorf("bounds ended = %v, want the wait's bound deleted", got)
|
|
||||||
}
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
|
||||||
t.Errorf("m = %+v, want failed naming the member and nothing built", v)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) {
|
|
||||||
r := memberRig(t, false)
|
|
||||||
mk, _ := builds.BuilderBuildKey("p", fpP)
|
|
||||||
data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
r.st.put(mk, data)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
|
||||||
t.Errorf("m = %+v, want failed naming the member and its error", v)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeMemberBounds keeps the bounds a builder arms and ends.
|
|
||||||
type fakeMemberBounds struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
arms, ends_ []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.arms = append(f.arms, key+" "+bound.String())
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeMemberBounds) End(_ context.Context, key string) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.ends_ = append(f.ends_, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms }
|
|
||||||
func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ }
|
|
||||||
|
|
||||||
func jsonEqual(a, b any) bool {
|
|
||||||
x, _ := json.Marshal(a)
|
|
||||||
y, _ := json.Marshal(b)
|
|
||||||
return string(x) == string(y)
|
|
||||||
}
|
|
||||||
@@ -2,45 +2,19 @@ package build
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
|
||||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait
|
// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait
|
||||||
// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4).
|
// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4).
|
||||||
|
|
||||||
func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3
|
|
||||||
r := newRig(t, goBundle, archiveA)
|
|
||||||
trunkModule(t, r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildDone {
|
|
||||||
t.Fatalf("build = %+v, want done", v)
|
|
||||||
}
|
|
||||||
if ok, missing := v.Referenced(); !ok {
|
|
||||||
t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts)
|
|
||||||
}
|
|
||||||
rev := r.st.revs[r.key]
|
|
||||||
r.bm.mu.Lock()
|
|
||||||
r.bm.calls = nil
|
|
||||||
r.bm.mu.Unlock()
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint
|
|
||||||
r.b.Wait()
|
|
||||||
if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev {
|
|
||||||
t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2
|
func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2
|
||||||
r := newRig(t, goBundle)
|
r := newRig(t, goBundle)
|
||||||
r.bm.noRef["ace/tool"] = true
|
r.bm.noRef["ace/tool"] = true
|
||||||
@@ -63,7 +37,7 @@ func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key
|
func TestATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a)
|
||||||
r := newRig(t, goBundle)
|
r := newRig(t, goBundle)
|
||||||
r.consider(t, "modules/m")
|
r.consider(t, "modules/m")
|
||||||
for _, c := range r.bm.callsOf("build_go_build") {
|
for _, c := range r.bm.callsOf("build_go_build") {
|
||||||
@@ -71,15 +45,6 @@ func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's
|
|||||||
t.Errorf("trunk build args = %+v, want publish", c.body)
|
t.Errorf("trunk build args = %+v, want publish", c.body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
c := newRig(t, goBundle)
|
|
||||||
trunkModule(t, c)
|
|
||||||
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
c.b.Wait()
|
|
||||||
for _, call := range c.bm.callsOf("build_go_build") {
|
|
||||||
if _, ok := call.body.(map[string]any)["publish"]; ok {
|
|
||||||
t.Errorf("candidate build args = %+v, want no publish", call.body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// putBuild writes a builder key's value as the store would hold it.
|
// putBuild writes a builder key's value as the store would hold it.
|
||||||
@@ -164,25 +129,10 @@ func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4
|
|||||||
|
|
||||||
var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"}
|
var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"}
|
||||||
|
|
||||||
func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
|
func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublished(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
|
||||||
c := newRig(t, tsPackage)
|
|
||||||
c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
|
||||||
trunkModule(t, c)
|
|
||||||
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
c.b.Wait()
|
|
||||||
v := valueOf(t, c.st, c.key)
|
|
||||||
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" {
|
|
||||||
t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v)
|
|
||||||
}
|
|
||||||
for _, call := range c.bm.callsOf("build_node_build") {
|
|
||||||
if _, ok := call.body.(map[string]any)["publish"]; ok {
|
|
||||||
t.Errorf("a candidate's package build was asked to publish: %+v", call.body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
r := newRig(t, tsPackage)
|
r := newRig(t, tsPackage)
|
||||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
||||||
v = r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" {
|
if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" {
|
||||||
t.Fatalf("trunk build = %+v, want done naming the published version", v)
|
t.Fatalf("trunk build = %+v, want done naming the published version", v)
|
||||||
}
|
}
|
||||||
@@ -207,50 +157,6 @@ func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6
|
|
||||||
r := newRig(t, tsPackage, tsBundle)
|
|
||||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
|
||||||
trunkModule(t, r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
|
||||||
t.Fatalf("candidate build = %+v", v)
|
|
||||||
}
|
|
||||||
rev := r.st.revs[r.key]
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if r.st.revs[r.key] <= rev {
|
|
||||||
t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev)
|
|
||||||
}
|
|
||||||
if ok, missing := v.Referenced(); !ok {
|
|
||||||
t.Fatalf("build = %+v, missing references %v", v, missing)
|
|
||||||
}
|
|
||||||
rev = r.st.revs[r.key]
|
|
||||||
r.b.tried = map[string]bool{}
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
if r.st.revs[r.key] != rev {
|
|
||||||
t.Error("a published package was built again: once only")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6
|
|
||||||
r := newRig(t, tsPackage)
|
|
||||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
|
||||||
trunkModule(t, r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
for _, n := range []string{"ace", "novox"} {
|
|
||||||
r.bm.fails[n+"/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
|
|
||||||
}
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) {
|
|
||||||
t.Errorf("build = %+v, want failed opening %q", v, builds.VersionTaken)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref.
|
// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref.
|
||||||
func sdkModule(t *testing.T, r *rig, ref string) []byte {
|
func sdkModule(t *testing.T, r *rig, ref string) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -365,30 +271,6 @@ func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths
|
|
||||||
r := newRig(t, tsPackage)
|
|
||||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
|
||||||
trunkModule(t, r)
|
|
||||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m"},
|
|
||||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}}
|
|
||||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
|
|
||||||
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", data)
|
|
||||||
r.b.Wait()
|
|
||||||
f := r.bm.callsOf("build_node_fetch")
|
|
||||||
if len(f) == 0 {
|
|
||||||
t.Fatal("no fetch")
|
|
||||||
}
|
|
||||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
|
||||||
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
|
||||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
|
||||||
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
|
// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
|
||||||
// answered a digest alone (a build module from before R4e).
|
// answered a digest alone (a build module from before R4e).
|
||||||
func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
|
func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
|
||||||
|
|||||||
@@ -77,14 +77,9 @@ type Builder struct {
|
|||||||
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
|
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
|
||||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
|
||||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
|
||||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
|
||||||
// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
|
// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
|
||||||
// §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a
|
// §373, §465): the part a resolved package names in a build's dependencies. Nil names none.
|
||||||
// member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none.
|
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
|
||||||
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
|
|
||||||
MemberBounds MemberBounds
|
|
||||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||||
@@ -132,8 +127,6 @@ type Builder struct {
|
|||||||
again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built
|
again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built
|
||||||
plans map[string]planned // by manifest digest
|
plans map[string]planned // by manifest digest
|
||||||
stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go)
|
stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go)
|
||||||
cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go)
|
|
||||||
via map[string]via // by build key: a candidate's fetch (candidates.go)
|
|
||||||
// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
|
// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
|
||||||
outOfStep map[string][]string
|
outOfStep map[string][]string
|
||||||
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
|
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
|
||||||
@@ -141,10 +134,6 @@ type Builder struct {
|
|||||||
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
||||||
// waiting for one is built once it is here (ADR 0500 clause 6).
|
// waiting for one is built once it is here (ADR 0500 clause 6).
|
||||||
published map[string]bool
|
published map[string]bool
|
||||||
// memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired
|
|
||||||
// those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go).
|
|
||||||
memberWaits, memberExpired map[string]bool
|
|
||||||
fetched map[string][]sdkstep.FetchBuild
|
|
||||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||||
held map[string]bool
|
held map[string]bool
|
||||||
@@ -181,13 +170,12 @@ func (b *Builder) init() {
|
|||||||
limit = 256
|
limit = 256
|
||||||
}
|
}
|
||||||
b.sem = make(chan struct{}, limit)
|
b.sem = make(chan struct{}, limit)
|
||||||
b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{}
|
b.trunkFP = map[string]bool{}
|
||||||
b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{}
|
b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{}
|
||||||
b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{}
|
b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{}
|
||||||
b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
|
b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
|
||||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||||
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
|
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
|
||||||
b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{}
|
|
||||||
b.builds = map[string]context.CancelCauseFunc{}
|
b.builds = map[string]context.CancelCauseFunc{}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -265,7 +253,6 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
|||||||
b.trunkFP[bk] = true
|
b.trunkFP[bk] = true
|
||||||
b.origin[bk] = o
|
b.origin[bk] = o
|
||||||
b.edges[bk] = e
|
b.edges[bk] = e
|
||||||
delete(b.via, bk)
|
|
||||||
}
|
}
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
if superseded != nil {
|
if superseded != nil {
|
||||||
@@ -472,23 +459,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
|||||||
for _, k := range sortedKeys(known) {
|
for _, k := range sortedKeys(known) {
|
||||||
b.consider(ctx, k, known[k])
|
b.consider(ctx, k, known[k])
|
||||||
}
|
}
|
||||||
b.TrunkMoved(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
|
||||||
// revision it was previewed onto had no trunk head to be merged onto.
|
|
||||||
func (b *Builder) TrunkMoved(ctx context.Context) {
|
|
||||||
b.init()
|
|
||||||
b.mu.Lock()
|
|
||||||
cands := make([]meshcatalog.CandidatesByGroupValue, 0, len(b.cands))
|
|
||||||
for _, c := range b.cands {
|
|
||||||
cands = append(cands, c)
|
|
||||||
}
|
|
||||||
b.mu.Unlock()
|
|
||||||
sort.Slice(cands, func(i, j int) bool { return cands[i].Group < cands[j].Group })
|
|
||||||
for _, c := range cands {
|
|
||||||
b.considerCandidate(ctx, c)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string {
|
func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string {
|
||||||
@@ -630,13 +600,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
memberFail := ""
|
|
||||||
if !trunk && planErr == nil && !nothing {
|
|
||||||
var waiting bool
|
|
||||||
if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting {
|
|
||||||
return // a package member of the group is not built (design 47 G §373): its end looks at this key again
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||||
if reserving && b.waitFill(module, m) {
|
if reserving && b.waitFill(module, m) {
|
||||||
@@ -654,7 +617,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
b.handClaimed(key, cl.Rev)
|
b.handClaimed(key, cl.Rev)
|
||||||
b.logBuild(LogStarted(module, m, b.refOf(module, m.Fp2), b.Node, b.now()))
|
b.logBuild(LogStarted(module, m, "", b.Node, b.now()))
|
||||||
if cl.Unasked != nil {
|
if cl.Unasked != nil {
|
||||||
b.logf("%v", cl.Unasked)
|
b.logf("%v", cl.Unasked)
|
||||||
}
|
}
|
||||||
@@ -686,8 +649,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
|||||||
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
|
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
|
||||||
// fails with why, and is taken again at the next look.
|
// fails with why, and is taken again at the next look.
|
||||||
result = failed("%s", numbering.Refused)
|
result = failed("%s", numbering.Refused)
|
||||||
} else if memberFail != "" && cl.Build.State != builds.BuildDone {
|
|
||||||
result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373)
|
|
||||||
} else if nothing {
|
} else if nothing {
|
||||||
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
|
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
|
||||||
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
|
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
|
||||||
@@ -734,8 +695,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
|||||||
if b.Ended != nil {
|
if b.Ended != nil {
|
||||||
b.Ended(ctx, cl.Key)
|
b.Ended(ctx, cl.Key)
|
||||||
}
|
}
|
||||||
b.memberEnded(ctx, cl.Key)
|
ended := LogEnded(module, m, "", b.Node, b.now(), result)
|
||||||
ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result)
|
|
||||||
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
|
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
|
||||||
b.logBuild(ended)
|
b.logBuild(ended)
|
||||||
if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
|
if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
|
||||||
@@ -1074,15 +1034,6 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
|||||||
// key taken over at start waits for the ask the key names and puts none.
|
// key taken over at start waits for the ask the key names and puts none.
|
||||||
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) {
|
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) {
|
||||||
req := sdkstep.Fetch{Repository: repository, Commit: m.Commit}
|
req := sdkstep.Fetch{Repository: repository, Commit: m.Commit}
|
||||||
if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
|
|
||||||
b.mu.Lock()
|
|
||||||
v, candidate := b.via[key]
|
|
||||||
b.mu.Unlock()
|
|
||||||
if candidate {
|
|
||||||
req.Ref, req.Base = v.Ref, v.Base
|
|
||||||
req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
|
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
|
||||||
// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
|
// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
|
||||||
// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
|
// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
|
||||||
@@ -1092,17 +1043,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
|||||||
}
|
}
|
||||||
sort.Strings(req.Paths)
|
sort.Strings(req.Paths)
|
||||||
}
|
}
|
||||||
candidate := req.Base != ""
|
|
||||||
seat, _ := b.Seats.OfModule(l.module)
|
seat, _ := b.Seats.OfModule(l.module)
|
||||||
language := seat.AskLanguage()
|
language := seat.AskLanguage()
|
||||||
kind := sdkstep.AskDelivery
|
kind := sdkstep.AskDelivery
|
||||||
if candidate {
|
|
||||||
kind = sdkstep.AskCheck
|
|
||||||
}
|
|
||||||
job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain),
|
job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain),
|
||||||
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
|
Publish: true}
|
||||||
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
|
|
||||||
Publish: !candidate}
|
|
||||||
for _, t := range l.tasks {
|
for _, t := range l.tasks {
|
||||||
a := sdkstep.AskArtifact{Artifact: t.Artifact}
|
a := sdkstep.AskArtifact{Artifact: t.Artifact}
|
||||||
if t.Kind == catalogue.ArtifactUpstream {
|
if t.Kind == catalogue.ArtifactUpstream {
|
||||||
@@ -1197,7 +1142,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
|||||||
}
|
}
|
||||||
l.digests[t.Artifact] = a.Digest
|
l.digests[t.Artifact] = a.Digest
|
||||||
l.resolved[t.Artifact] = a.Resolved
|
l.resolved[t.Artifact] = a.Resolved
|
||||||
if a.Reference != "" && (stored(t.Kind) || !candidate) {
|
if a.Reference != "" {
|
||||||
l.references[t.Artifact] = a.Reference
|
l.references[t.Artifact] = a.Reference
|
||||||
}
|
}
|
||||||
if a.Reference != "" && b.Answered != nil {
|
if a.Reference != "" && b.Answered != nil {
|
||||||
@@ -1236,17 +1181,6 @@ func (b *Builder) logBuild(e builds.BuildLogEntry) {
|
|||||||
b.Log(e)
|
b.Log(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
|
|
||||||
func (b *Builder) refOf(module, fp string) string {
|
|
||||||
key, err := builds.BuilderBuildKey(module, fp)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
return b.via[key].Ref
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
|
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
|
||||||
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
|
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
|
||||||
// that asked it and the time.
|
// that asked it and the time.
|
||||||
|
|||||||
@@ -187,7 +187,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
|||||||
r.b = &Builder{
|
r.b = &Builder{
|
||||||
Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||||
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
if r.key, err = builds.BuilderBuildKey("m", fp1); err != nil {
|
if r.key, err = builds.BuilderBuildKey("m", fp1); err != nil {
|
||||||
@@ -437,31 +436,6 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision
|
|
||||||
r := newRig(t, imageArt)
|
|
||||||
r.puller = "ace"
|
|
||||||
trunkModule(t, r)
|
|
||||||
// Design 47 G §373: a candidate's build is numbered under no tag, so it numbers nothing.
|
|
||||||
nums := &fakeNumbers{}
|
|
||||||
r.b.Numbers = nums
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
if nums.numbered != 0 {
|
|
||||||
t.Errorf("a candidate's build was numbered %d times, want none", nums.numbered)
|
|
||||||
}
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" {
|
|
||||||
t.Fatalf("build = %+v, want done with no reference", v)
|
|
||||||
}
|
|
||||||
calls := r.bm.callsOf("build_image_build")
|
|
||||||
if len(calls) != 1 {
|
|
||||||
t.Fatalf("%d image builds, want 1", len(calls))
|
|
||||||
}
|
|
||||||
if _, has := calls[0].body.(map[string]any)["publish"]; has {
|
|
||||||
t.Errorf("a candidate's image call carries publish: %+v", calls[0].body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim
|
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.puller = "ace"
|
r.puller = "ace"
|
||||||
@@ -793,81 +767,6 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5
|
|
||||||
r := newRig(t, imageArt)
|
|
||||||
r.puller = "ace"
|
|
||||||
trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildDone || v.Artifacts[0].Reference != "" {
|
|
||||||
t.Fatalf("candidate build = %+v, want done with no reference", v)
|
|
||||||
}
|
|
||||||
if n := len(r.bm.callsOf("build_image_build")); n != 1 {
|
|
||||||
t.Fatalf("%d image builds after the candidate, want 1", n)
|
|
||||||
}
|
|
||||||
// The pull request is merged: the trunk's record names the candidate's fingerprint.
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
v = valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" {
|
|
||||||
t.Fatalf("trunk build = %+v, want done with a reference", v)
|
|
||||||
}
|
|
||||||
calls := r.bm.callsOf("build_image_build")
|
|
||||||
published := 0
|
|
||||||
for _, c := range calls {
|
|
||||||
if c.body.(map[string]any)["publish"] == true {
|
|
||||||
published++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(calls) != 2 || published != 1 {
|
|
||||||
t.Errorf("%d image builds, %d with publish, want 2 and exactly 1", len(calls), published)
|
|
||||||
}
|
|
||||||
// Written once more only: looking again builds nothing.
|
|
||||||
rev := r.st.revs[r.key]
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_image_build")) != 2 {
|
|
||||||
t.Error("a published key was built again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
|
|
||||||
r := newRig(t, imageArt)
|
|
||||||
r.puller = "ace"
|
|
||||||
trunkModule(t, r)
|
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
|
||||||
t.Fatalf("candidate build = %+v", v)
|
|
||||||
}
|
|
||||||
r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused"
|
|
||||||
// Issue 1748: a run of the ask that ended on a transport fault is said in the log and on the key, and is no end.
|
|
||||||
inner, log := r.asks.Answer, &logged{}
|
|
||||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
|
||||||
r.asks.Write(buildstep.Step{ID: a.ID, Node: "g14", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Reason: "lookup registry: i/o timeout"})
|
|
||||||
return inner(a)
|
|
||||||
}
|
|
||||||
r.b.Logf = log.logf
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint
|
|
||||||
r.b.Wait()
|
|
||||||
r.asks.Answer, r.b.Logf = inner, t.Logf
|
|
||||||
if !log.has("run 1 of", "on g14 ended on a transport fault", "lookup registry: i/o timeout") {
|
|
||||||
t.Errorf("a run given again was not said: %v", log.lines)
|
|
||||||
}
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != builds.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") {
|
|
||||||
t.Fatalf("build = %+v, want failed naming the publish and its reason", v)
|
|
||||||
}
|
|
||||||
// The failed key is retaken as any failed key is, and ends published once the registry answers.
|
|
||||||
delete(r.bm.fails, "ace/build_image_build")
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Artifacts[0].Reference == "" {
|
|
||||||
t.Errorf("build = %+v, want done with a reference", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||||
// drop from their stream (Dropper).
|
// drop from their stream (Dropper).
|
||||||
type goneAsks struct {
|
type goneAsks struct {
|
||||||
|
|||||||
@@ -73,11 +73,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
|||||||
}
|
}
|
||||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path,
|
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path,
|
||||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages}
|
Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages}
|
||||||
b.mu.Lock()
|
|
||||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
|
||||||
b.via[key] = via{}
|
|
||||||
}
|
|
||||||
b.mu.Unlock()
|
|
||||||
b.consider(ctx, entry.Module, m)
|
b.consider(ctx, entry.Module, m)
|
||||||
select {
|
select {
|
||||||
case log := <-ended:
|
case log := <-ended:
|
||||||
|
|||||||
@@ -38,16 +38,8 @@ const (
|
|||||||
oldFormBoundPrefix = "old-form."
|
oldFormBoundPrefix = "old-form."
|
||||||
unputBoundPrefix = "unput."
|
unputBoundPrefix = "unput."
|
||||||
stepsBoundPrefix = "steps."
|
stepsBoundPrefix = "steps."
|
||||||
memberBoundPrefix = "member."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77):
|
|
||||||
// `member.<build key>`. Its end while the member is not built fails the dependent's step naming the member.
|
|
||||||
func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey }
|
|
||||||
|
|
||||||
// MemberBoundOf is the build key a member wait's bound names, and whether key is one.
|
|
||||||
func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) }
|
|
||||||
|
|
||||||
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
|
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
|
||||||
// novox/hq ADR 0668 §3): `unput.<build key>`. Its end while the key still stands so, with no look holding it, is the
|
// novox/hq ADR 0668 §3): `unput.<build key>`. Its end while the key still stands so, with no look holding it, is the
|
||||||
// finding call-unanswered about the key.
|
// finding call-unanswered about the key.
|
||||||
|
|||||||
Reference in New Issue
Block a user