From efcc8125af67e93c269aa1ea9da0eb6b44d986b9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:09:33 +0100 Subject: [PATCH] BL9: a group run's build names each package member's build by its digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A candidate's module is fetched against each package member of its group by the digest of the member's build (fetch `builds`), the member out of the fetched packages; a member not built holds it with wait-held naming the member, its bound a key of the builder's bounds state with its own time to live (A-77), a failed member or one missing past the bound fails the step naming it. Builds: design 47 G §373, §465 (A-77) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/main.go | 8 + .../cmd/mesh-delivery-builder/memberbounds.go | 42 ++++ .../internal/build/candidates.go | 203 +++++++++++++++++- .../internal/build/members_test.go | 166 ++++++++++++++ .../internal/build/run.go | 24 +++ .../internal/versions/names.go | 32 +++ .../internal/versions/publish.go | 11 +- 7 files changed, 483 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go create mode 100644 modules/mesh-delivery-builder/internal/build/members_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b873b4c2..adac30c3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }, errs: errs, logf: logger.Printf} trunks.start() b.Trunk = trunks.head + b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} + b.Members = versions.ModuleOf(indexerPackages{packages}) // the build of a group's package member (design 47 G §373, §465) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the // indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since. b.Log = func(e builds.BuildLogEntry) { @@ -419,6 +421,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } + if key, ok := versions.MemberBoundOf(e.Key()); ok { + if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member + b.MemberBoundEnded(ctx, key) + } + return + } if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended reads.expiredCall(node) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go new file mode 100644 index 00000000..63e15b10 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/versions" +) + +// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of +// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded). +// No timer. +type memberBounds struct { + bounds func() jetstream.KeyValue + now func() time.Time + logf func(string, ...any) +} + +// Arm is build.MemberBounds' Arm. +func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error { + kv := m.bounds() + if kv == nil { + return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } + if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { + return err + } + return nil +} + +// End is build.MemberBounds' End. +func (m *memberBounds) End(ctx context.Context, key string) { + if kv := m.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + m.logf("the member bound of %s is not deleted: %v", key, err) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index fc65815c..2cf7d3e4 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -3,7 +3,11 @@ package build import ( "context" "fmt" + "maps" + "slices" "sort" + "strings" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -20,7 +24,15 @@ import ( // via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk // commit the head is merged onto. -type via struct{ Ref, Base string } +type via struct { + Ref, Base string + // Members are the group's package members the module is built against by their builds (design 47 G §373, §465). + Members []member +} + +// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's +// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate. +type member struct{ Package, Module, Artifact, Fp string } // Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it // carries. A value that is not a candidate is said and left. @@ -46,6 +58,14 @@ func (b *Builder) ForgetCandidate(key string) { } func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) { + group := map[string]string{} // the fingerprint of each module the group's pull requests change + for _, p := range c.Pulls { + for name, cm := range p.Fps { + if cm.Fp2 != "" && cm.Manifest != "" { + group[name] = cm.Fp2 + } + } + } for _, p := range c.Pulls { names := make([]string, 0, len(p.Fps)) for name := range p.Fps { @@ -83,9 +103,14 @@ func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.Candidate b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) continue } + members, err := b.membersOf(ctx, name, cm, group) + if err != nil { + b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) + continue + } b.mu.Lock() if _, onTrunk := b.trunkFP[key]; !onTrunk { - b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base} + b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members} } b.mu.Unlock() b.consider(ctx, name, m) @@ -131,3 +156,177 @@ func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) } return head, nil } + +// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package +// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the +// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds, +// is no member: it is fetched as before. +func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) { + if b.Members == nil { + return nil, nil + } + var out []member + for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) { + if len(cm.Packages[pkg]) != 40 { + continue + } + module, artifact, found, err := b.Members(ctx, pkg) + if err != nil { + return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err) + } + if !found || module == name || group[module] == "" { + continue + } + out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]}) + } + return out, nil +} + +// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's +// 60-minute bound). +const MemberBound = 60 * time.Minute + +// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77): +// Arm writes it for key, the dependent's build key; End deletes it. +type MemberBounds interface { + Arm(ctx context.Context, key, why string, bound time.Duration) error + End(ctx context.Context, key string) +} + +// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the +// candidates are looked at again, which fails its step naming the member still not built. +func (b *Builder) MemberBoundEnded(ctx context.Context, key string) { + b.init() + b.mu.Lock() + waiting := b.memberWaits[key] + if waiting { + b.memberExpired[key] = true + } + b.mu.Unlock() + if waiting { + b.TrunkMoved(ctx) + } +} + +// endMemberWait forgets key's wait and deletes its bound. +func (b *Builder) endMemberWait(ctx context.Context, key string) { + b.mu.Lock() + had := b.memberWaits[key] + delete(b.memberWaits, key) + delete(b.memberExpired, key) + b.mu.Unlock() + if had && b.MemberBounds != nil { + b.MemberBounds.End(ctx, key) + } +} + +// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the +// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or +// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the +// builder holds no timer. +func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) { + b.mu.Lock() + v := b.via[key] + b.mu.Unlock() + if len(v.Members) == 0 { + return nil, "", false + } + defer func() { + if !waiting { + b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound + } + }() + var pending []string + for _, mem := range v.Members { + ref := mem.Module + "/" + mem.Artifact + mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp) + if err != nil { + return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false + } + raw, _, found, err := b.Claimer.Store.Get(ctx, mk) + if err != nil { + return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false + } + if !found { + pending = append(pending, ref) + continue + } + cur, err := builds.DecodeBuilderBuild(raw) + if err != nil { + return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false + } + switch cur.State { + case builds.BuildDone: + digest := "" + for _, a := range cur.Artifacts { + if a.Name == mem.Artifact { + digest = a.Digest + } + } + if digest == "" { + return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false + } + got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest}) + case builds.BuildFailed: + return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false + default: + pending = append(pending, ref) + } + } + if len(pending) == 0 { + b.mu.Lock() + b.fetched[key] = got + b.mu.Unlock() + return got, "", false + } + b.mu.Lock() + said := b.memberWaits[key] + expired := b.memberExpired[key] + b.mu.Unlock() + if expired { + return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false + } + if !said { + line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String() + if b.MemberBounds != nil { + // The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written + // would never end, so the step fails at once, saying so. + if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil { + return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false + } + } + b.mu.Lock() + b.memberWaits[key] = true + b.mu.Unlock() + b.logf("%s", line) + if b.Log != nil { + b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"}) + } + } + return nil, "", true +} + +// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's +// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits. +func (b *Builder) memberEnded(ctx context.Context, key string) { + b.mu.Lock() + member := false + for _, v := range b.via { + for _, mem := range v.Members { + if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key { + member = true + } + } + } + b.mu.Unlock() + if member { + b.TrunkMoved(ctx) + } +} + +// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none. +func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild { + b.mu.Lock() + defer b.mu.Unlock() + return b.fetched[key] +} diff --git a/modules/mesh-delivery-builder/internal/build/members_test.go b/modules/mesh-delivery-builder/internal/build/members_test.go new file mode 100644 index 00000000..7354e477 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/members_test.go @@ -0,0 +1,166 @@ +package build + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const memberPackage = "example.org/p" + +var fpP = "src2:" + hex64('2') + +// groupValue is a group's candidate: the module m, built against the head of the package member p, and p. +func groupValue(t *testing.T) []byte { + t.Helper() + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{ + "m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}}, + "p": {Fp2: fpP, Manifest: mdigest('b')}}}} + hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}}) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + return data +} + +// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known. +func memberRig(t *testing.T, known bool) *rig { + t.Helper() + r := newRig(t, goBundle) + lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"} + r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}} + r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) { + if pkg == memberPackage { + return "p", "lib", true, nil + } + return "", "", false, nil + } + trunkModule(t, r) + if known { + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'), + Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "p", data) + r.b.Wait() + } + return r +} + +func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103 + t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) { + r := memberRig(t, true) + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { + t.Fatalf("m = %+v, want built", v) + } + var found bool + for _, c := range r.bm.callsOf("build_go_fetch") { + f := c.body.(map[string]any) + bs, _ := f["builds"].([]any) + if len(bs) == 0 { + continue + } + found = true + want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest} + if len(bs) != 1 || !jsonEqual(bs[0], want) { + t.Errorf("builds = %v, want %v", bs, want) + } + pk, _ := f["packages"].(map[string]any) + if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" { + t.Errorf("packages = %v, want the member out of it and the other package kept", pk) + } + } + if !found { + t.Error("no fetch named the member's build") + } + }) + t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) { + r := memberRig(t, false) + var mu sync.Mutex + bounds := &fakeMemberBounds{} + r.b.MemberBounds = bounds + var said []builds.BuildLogEntry + r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() } + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Fatal("the dependent was claimed or built while its package member is not built") + } + mu.Lock() + if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") { + t.Errorf("said = %+v, want one wait-held naming p/lib", said) + } + mu.Unlock() + if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() { + t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got) + } + r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry + r.b.Wait() + if got := bounds.ended(); len(got) != 1 || got[0] != r.key { + t.Errorf("bounds ended = %v, want the wait's bound deleted", got) + } + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Errorf("m = %+v, want failed naming the member and nothing built", v) + } + }) + t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) { + r := memberRig(t, false) + mk, _ := builds.BuilderBuildKey("p", fpP) + data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"}) + if err != nil { + t.Fatal(err) + } + r.st.put(mk, data) + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Errorf("m = %+v, want failed naming the member and its error", v) + } + }) +} + +// fakeMemberBounds keeps the bounds a builder arms and ends. +type fakeMemberBounds struct { + mu sync.Mutex + arms, ends_ []string +} + +func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error { + f.mu.Lock() + defer f.mu.Unlock() + f.arms = append(f.arms, key+" "+bound.String()) + return nil +} + +func (f *fakeMemberBounds) End(_ context.Context, key string) { + f.mu.Lock() + defer f.mu.Unlock() + f.ends_ = append(f.ends_, key) +} + +func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms } +func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ } + +func jsonEqual(a, b any) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ff63d787..52402be3 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -74,6 +74,11 @@ type Builder struct { // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. Trunk func(ctx context.Context, rev uint64, repository string) (string, error) + // Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G + // §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a + // member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none. + Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error) + MemberBounds MemberBounds // Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. @@ -125,6 +130,10 @@ type Builder struct { // published are the package versions `@` a done key of this bucket names as a package's reference: a key // waiting for one is built once it is here (ADR 0500 clause 6). published map[string]bool + // memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired + // those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go). + memberWaits, memberExpired map[string]bool + fetched map[string][]sdkstep.FetchBuild // held are the done keys' references the store answered held, by ` `, asked once per process: a // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). held map[string]bool @@ -163,6 +172,7 @@ func (b *Builder) init() { b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} + b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{} b.builds = map[string]context.CancelCauseFunc{} }) } @@ -501,6 +511,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } } + memberFail := "" + if !trunk && planErr == nil && !nothing { + var waiting bool + if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting { + return // a package member of the group is not built (design 47 G §373): its end looks at this key again + } + } reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { return @@ -549,6 +566,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB // Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key // fails with why, and is taken again at the next look. result = failed("%s", numbering.Refused) + } else if memberFail != "" && cl.Build.State != builds.BuildDone { + result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373) } else if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). @@ -595,6 +614,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB if b.Ended != nil { b.Ended(ctx, cl.Key) } + b.memberEnded(ctx, cl.Key) ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result) ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26) b.logBuild(ended) @@ -932,6 +952,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes b.mu.Unlock() if candidate { req.Ref, req.Base = v.Ref, v.Base + req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465) } } if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 { @@ -968,6 +989,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) + for _, fb := range req.Builds { + delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) + } } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go index 66dd54d0..e6e7d091 100644 --- a/modules/mesh-delivery-builder/internal/versions/names.go +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -70,3 +70,35 @@ func isSeriesKey(key string) bool { _, _, ok := meshcatalog.ParsePackagesByPackageSeriesKey(key) return ok } + +// ModuleOf is the module and artifact a package's name in its ecosystem is built from, as the indexer's packages state holds +// them (the inverse of NamesIn; design 47 G §373, §465): the series key whose value names that name. +func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { + return func(ctx context.Context, name string) (string, string, bool, error) { + keys, err := s.Keys(ctx) + if err != nil { + return "", "", false, err + } + sort.Strings(keys) + for _, key := range keys { + if !isSeriesKey(key) { + continue + } + value, found, err := s.Get(ctx, key) + if err != nil { + return "", "", false, err + } + if !found { + continue + } + v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) + if err != nil || v.Name != name { + continue + } + if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" { + return module, artifact, true, nil + } + } + return "", "", false, nil + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index 7b401b8a..7384de9c 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -37,8 +37,16 @@ const ( publishBoundPrefix = "publish." oldFormBoundPrefix = "old-form." unputBoundPrefix = "unput." + memberBoundPrefix = "member." ) +// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): +// `member.`. Its end while the member is not built fails the dependent's step naming the member. +func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey } + +// MemberBoundOf is the build key a member wait's bound names, and whether key is one. +func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) } + // UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, // novox/hq ADR 0668 §3): `unput.`. Its end while the key still stands so, with no look holding it, is the // finding call-unanswered about the key. @@ -68,7 +76,8 @@ const BoundsTool = "builder_bounds" const BoundsDescription = "The builder's expiring keys, its state " + BoundsState + " (design 51 *Health and conditions*): each key with what it " + "bounds (a version's publish, whose end unanswered is the finding " + PublishUnanswered + "; or a node's seven days holding the old " + "form's window, whose end is the finding " + OldFormHeld + "; or a build key left building because its end could not be written, " + - "whose end is the finding call-unanswered), when it ends, and how long is left. The bus ends each key by its own time to live." + "whose end is the finding call-unanswered; or a dependent's wait for a package member of a group's run, whose end fails its " + + "step), when it ends, and how long is left. The bus ends each key by its own time to live." // Bound is a key's value in BoundsState: what it bounds, in plain words, and when it ends. type Bound struct {