BL9: a group run's build names each package member's build by its digest
A candidate's module is fetched against each package member of its group by the digest of the member's build (fetch `builds`), the member out of the fetched packages; a member not built holds it with wait-held naming the member, its bound a key of the builder's bounds state with its own time to live (A-77), a failed member or one missing past the bound fails the step naming it. Builds: design 47 G §373, §465 (A-77) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
|
||||
b.Members = versions.ModuleOf(indexerPackages{packages}) // the build of a group's package member (design 47 G §373, §465)
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||
b.Log = func(e builds.BuildLogEntry) {
|
||||
@@ -419,6 +421,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
unput.expired(ctx, e)
|
||||
return
|
||||
}
|
||||
if key, ok := versions.MemberBoundOf(e.Key()); ok {
|
||||
if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member
|
||||
b.MemberBoundEnded(ctx, key)
|
||||
}
|
||||
return
|
||||
}
|
||||
if node, ok := enginereads.BoundOf(e.Key()); ok {
|
||||
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
|
||||
reads.expiredCall(node)
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of
|
||||
// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded).
|
||||
// No timer.
|
||||
type memberBounds struct {
|
||||
bounds func() jetstream.KeyValue
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// Arm is build.MemberBounds' Arm.
|
||||
func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error {
|
||||
kv := m.bounds()
|
||||
if kv == nil {
|
||||
return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
|
||||
}
|
||||
if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// End is build.MemberBounds' End.
|
||||
func (m *memberBounds) End(ctx context.Context, key string) {
|
||||
if kv := m.bounds(); kv != nil {
|
||||
if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
m.logf("the member bound of %s is not deleted: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,11 @@ package build
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
@@ -20,7 +24,15 @@ import (
|
||||
|
||||
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
|
||||
// commit the head is merged onto.
|
||||
type via struct{ Ref, Base string }
|
||||
type via struct {
|
||||
Ref, Base string
|
||||
// Members are the group's package members the module is built against by their builds (design 47 G §373, §465).
|
||||
Members []member
|
||||
}
|
||||
|
||||
// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's
|
||||
// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate.
|
||||
type member struct{ Package, Module, Artifact, Fp string }
|
||||
|
||||
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
|
||||
// carries. A value that is not a candidate is said and left.
|
||||
@@ -46,6 +58,14 @@ func (b *Builder) ForgetCandidate(key string) {
|
||||
}
|
||||
|
||||
func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) {
|
||||
group := map[string]string{} // the fingerprint of each module the group's pull requests change
|
||||
for _, p := range c.Pulls {
|
||||
for name, cm := range p.Fps {
|
||||
if cm.Fp2 != "" && cm.Manifest != "" {
|
||||
group[name] = cm.Fp2
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
names := make([]string, 0, len(p.Fps))
|
||||
for name := range p.Fps {
|
||||
@@ -83,9 +103,14 @@ func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.Candidate
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
|
||||
continue
|
||||
}
|
||||
members, err := b.membersOf(ctx, name, cm, group)
|
||||
if err != nil {
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
|
||||
continue
|
||||
}
|
||||
b.mu.Lock()
|
||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, name, m)
|
||||
@@ -131,3 +156,177 @@ func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package
|
||||
// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the
|
||||
// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds,
|
||||
// is no member: it is fetched as before.
|
||||
func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) {
|
||||
if b.Members == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out []member
|
||||
for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) {
|
||||
if len(cm.Packages[pkg]) != 40 {
|
||||
continue
|
||||
}
|
||||
module, artifact, found, err := b.Members(ctx, pkg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err)
|
||||
}
|
||||
if !found || module == name || group[module] == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's
|
||||
// 60-minute bound).
|
||||
const MemberBound = 60 * time.Minute
|
||||
|
||||
// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77):
|
||||
// Arm writes it for key, the dependent's build key; End deletes it.
|
||||
type MemberBounds interface {
|
||||
Arm(ctx context.Context, key, why string, bound time.Duration) error
|
||||
End(ctx context.Context, key string)
|
||||
}
|
||||
|
||||
// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the
|
||||
// candidates are looked at again, which fails its step naming the member still not built.
|
||||
func (b *Builder) MemberBoundEnded(ctx context.Context, key string) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
waiting := b.memberWaits[key]
|
||||
if waiting {
|
||||
b.memberExpired[key] = true
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if waiting {
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// endMemberWait forgets key's wait and deletes its bound.
|
||||
func (b *Builder) endMemberWait(ctx context.Context, key string) {
|
||||
b.mu.Lock()
|
||||
had := b.memberWaits[key]
|
||||
delete(b.memberWaits, key)
|
||||
delete(b.memberExpired, key)
|
||||
b.mu.Unlock()
|
||||
if had && b.MemberBounds != nil {
|
||||
b.MemberBounds.End(ctx, key)
|
||||
}
|
||||
}
|
||||
|
||||
// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the
|
||||
// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or
|
||||
// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the
|
||||
// builder holds no timer.
|
||||
func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) {
|
||||
b.mu.Lock()
|
||||
v := b.via[key]
|
||||
b.mu.Unlock()
|
||||
if len(v.Members) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
defer func() {
|
||||
if !waiting {
|
||||
b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound
|
||||
}
|
||||
}()
|
||||
var pending []string
|
||||
for _, mem := range v.Members {
|
||||
ref := mem.Module + "/" + mem.Artifact
|
||||
mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, mk)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false
|
||||
}
|
||||
if !found {
|
||||
pending = append(pending, ref)
|
||||
continue
|
||||
}
|
||||
cur, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false
|
||||
}
|
||||
switch cur.State {
|
||||
case builds.BuildDone:
|
||||
digest := ""
|
||||
for _, a := range cur.Artifacts {
|
||||
if a.Name == mem.Artifact {
|
||||
digest = a.Digest
|
||||
}
|
||||
}
|
||||
if digest == "" {
|
||||
return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false
|
||||
}
|
||||
got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest})
|
||||
case builds.BuildFailed:
|
||||
return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false
|
||||
default:
|
||||
pending = append(pending, ref)
|
||||
}
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
b.mu.Lock()
|
||||
b.fetched[key] = got
|
||||
b.mu.Unlock()
|
||||
return got, "", false
|
||||
}
|
||||
b.mu.Lock()
|
||||
said := b.memberWaits[key]
|
||||
expired := b.memberExpired[key]
|
||||
b.mu.Unlock()
|
||||
if expired {
|
||||
return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false
|
||||
}
|
||||
if !said {
|
||||
line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String()
|
||||
if b.MemberBounds != nil {
|
||||
// The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written
|
||||
// would never end, so the step fails at once, saying so.
|
||||
if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil {
|
||||
return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false
|
||||
}
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.memberWaits[key] = true
|
||||
b.mu.Unlock()
|
||||
b.logf("%s", line)
|
||||
if b.Log != nil {
|
||||
b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"})
|
||||
}
|
||||
}
|
||||
return nil, "", true
|
||||
}
|
||||
|
||||
// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's
|
||||
// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits.
|
||||
func (b *Builder) memberEnded(ctx context.Context, key string) {
|
||||
b.mu.Lock()
|
||||
member := false
|
||||
for _, v := range b.via {
|
||||
for _, mem := range v.Members {
|
||||
if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key {
|
||||
member = true
|
||||
}
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if member {
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none.
|
||||
func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.fetched[key]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
const memberPackage = "example.org/p"
|
||||
|
||||
var fpP = "src2:" + hex64('2')
|
||||
|
||||
// groupValue is a group's candidate: the module m, built against the head of the package member p, and p.
|
||||
func groupValue(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"},
|
||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{
|
||||
"m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}},
|
||||
"p": {Fp2: fpP, Manifest: mdigest('b')}}}}
|
||||
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known.
|
||||
func memberRig(t *testing.T, known bool) *rig {
|
||||
t.Helper()
|
||||
r := newRig(t, goBundle)
|
||||
lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"}
|
||||
r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}}
|
||||
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
|
||||
if pkg == memberPackage {
|
||||
return "p", "lib", true, nil
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
trunkModule(t, r)
|
||||
if known {
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'),
|
||||
Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "p", data)
|
||||
r.b.Wait()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103
|
||||
t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) {
|
||||
r := memberRig(t, true)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("m = %+v, want built", v)
|
||||
}
|
||||
var found bool
|
||||
for _, c := range r.bm.callsOf("build_go_fetch") {
|
||||
f := c.body.(map[string]any)
|
||||
bs, _ := f["builds"].([]any)
|
||||
if len(bs) == 0 {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest}
|
||||
if len(bs) != 1 || !jsonEqual(bs[0], want) {
|
||||
t.Errorf("builds = %v, want %v", bs, want)
|
||||
}
|
||||
pk, _ := f["packages"].(map[string]any)
|
||||
if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" {
|
||||
t.Errorf("packages = %v, want the member out of it and the other package kept", pk)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("no fetch named the member's build")
|
||||
}
|
||||
})
|
||||
t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) {
|
||||
r := memberRig(t, false)
|
||||
var mu sync.Mutex
|
||||
bounds := &fakeMemberBounds{}
|
||||
r.b.MemberBounds = bounds
|
||||
var said []builds.BuildLogEntry
|
||||
r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
||||
r.b.Wait()
|
||||
if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
||||
t.Fatal("the dependent was claimed or built while its package member is not built")
|
||||
}
|
||||
mu.Lock()
|
||||
if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") {
|
||||
t.Errorf("said = %+v, want one wait-held naming p/lib", said)
|
||||
}
|
||||
mu.Unlock()
|
||||
if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() {
|
||||
t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got)
|
||||
}
|
||||
r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry
|
||||
r.b.Wait()
|
||||
if got := bounds.ended(); len(got) != 1 || got[0] != r.key {
|
||||
t.Errorf("bounds ended = %v, want the wait's bound deleted", got)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
||||
t.Errorf("m = %+v, want failed naming the member and nothing built", v)
|
||||
}
|
||||
})
|
||||
t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) {
|
||||
r := memberRig(t, false)
|
||||
mk, _ := builds.BuilderBuildKey("p", fpP)
|
||||
data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.put(mk, data)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 {
|
||||
t.Errorf("m = %+v, want failed naming the member and its error", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fakeMemberBounds keeps the bounds a builder arms and ends.
|
||||
type fakeMemberBounds struct {
|
||||
mu sync.Mutex
|
||||
arms, ends_ []string
|
||||
}
|
||||
|
||||
func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.arms = append(f.arms, key+" "+bound.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeMemberBounds) End(_ context.Context, key string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.ends_ = append(f.ends_, key)
|
||||
}
|
||||
|
||||
func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms }
|
||||
func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ }
|
||||
|
||||
func jsonEqual(a, b any) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
@@ -74,6 +74,11 @@ type Builder struct {
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||
// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
|
||||
// §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a
|
||||
// member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none.
|
||||
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
|
||||
MemberBounds MemberBounds
|
||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||
@@ -125,6 +130,10 @@ type Builder struct {
|
||||
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
||||
// waiting for one is built once it is here (ADR 0500 clause 6).
|
||||
published map[string]bool
|
||||
// memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired
|
||||
// those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go).
|
||||
memberWaits, memberExpired map[string]bool
|
||||
fetched map[string][]sdkstep.FetchBuild
|
||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||
held map[string]bool
|
||||
@@ -163,6 +172,7 @@ func (b *Builder) init() {
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{}
|
||||
b.builds = map[string]context.CancelCauseFunc{}
|
||||
})
|
||||
}
|
||||
@@ -501,6 +511,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
return
|
||||
}
|
||||
}
|
||||
memberFail := ""
|
||||
if !trunk && planErr == nil && !nothing {
|
||||
var waiting bool
|
||||
if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting {
|
||||
return // a package member of the group is not built (design 47 G §373): its end looks at this key again
|
||||
}
|
||||
}
|
||||
reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if reserving && b.waitFill(module, m) {
|
||||
return
|
||||
@@ -549,6 +566,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
|
||||
// fails with why, and is taken again at the next look.
|
||||
result = failed("%s", numbering.Refused)
|
||||
} else if memberFail != "" && cl.Build.State != builds.BuildDone {
|
||||
result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373)
|
||||
} else if nothing {
|
||||
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
|
||||
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
|
||||
@@ -595,6 +614,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
if b.Ended != nil {
|
||||
b.Ended(ctx, cl.Key)
|
||||
}
|
||||
b.memberEnded(ctx, cl.Key)
|
||||
ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result)
|
||||
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
|
||||
b.logBuild(ended)
|
||||
@@ -932,6 +952,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
b.mu.Unlock()
|
||||
if candidate {
|
||||
req.Ref, req.Base = v.Ref, v.Base
|
||||
req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465)
|
||||
}
|
||||
}
|
||||
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
|
||||
@@ -968,6 +989,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
for _, fb := range req.Builds {
|
||||
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
|
||||
}
|
||||
}
|
||||
if id := asked.resume(language); id != "" {
|
||||
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||
|
||||
@@ -70,3 +70,35 @@ func isSeriesKey(key string) bool {
|
||||
_, _, ok := meshcatalog.ParsePackagesByPackageSeriesKey(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
// ModuleOf is the module and artifact a package's name in its ecosystem is built from, as the indexer's packages state holds
|
||||
// them (the inverse of NamesIn; design 47 G §373, §465): the series key whose value names that name.
|
||||
func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) {
|
||||
return func(ctx context.Context, name string) (string, string, bool, error) {
|
||||
keys, err := s.Keys(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if !isSeriesKey(key) {
|
||||
continue
|
||||
}
|
||||
value, found, err := s.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
|
||||
if err != nil || v.Name != name {
|
||||
continue
|
||||
}
|
||||
if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" {
|
||||
return module, artifact, true, nil
|
||||
}
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,16 @@ const (
|
||||
publishBoundPrefix = "publish."
|
||||
oldFormBoundPrefix = "old-form."
|
||||
unputBoundPrefix = "unput."
|
||||
memberBoundPrefix = "member."
|
||||
)
|
||||
|
||||
// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77):
|
||||
// `member.<build key>`. Its end while the member is not built fails the dependent's step naming the member.
|
||||
func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey }
|
||||
|
||||
// MemberBoundOf is the build key a member wait's bound names, and whether key is one.
|
||||
func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) }
|
||||
|
||||
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
|
||||
// novox/hq ADR 0668 §3): `unput.<build key>`. Its end while the key still stands so, with no look holding it, is the
|
||||
// finding call-unanswered about the key.
|
||||
@@ -68,7 +76,8 @@ const BoundsTool = "builder_bounds"
|
||||
const BoundsDescription = "The builder's expiring keys, its state " + BoundsState + " (design 51 *Health and conditions*): each key with what it " +
|
||||
"bounds (a version's publish, whose end unanswered is the finding " + PublishUnanswered + "; or a node's seven days holding the old " +
|
||||
"form's window, whose end is the finding " + OldFormHeld + "; or a build key left building because its end could not be written, " +
|
||||
"whose end is the finding call-unanswered), when it ends, and how long is left. The bus ends each key by its own time to live."
|
||||
"whose end is the finding call-unanswered; or a dependent's wait for a package member of a group's run, whose end fails its " +
|
||||
"step), when it ends, and how long is left. The bus ends each key by its own time to live."
|
||||
|
||||
// Bound is a key's value in BoundsState: what it bounds, in plain words, and when it ends.
|
||||
type Bound struct {
|
||||
|
||||
Reference in New Issue
Block a user