diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 47b89d0e..cdba8de9 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -251,8 +251,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro queueKV.Ready(again.bound(ctx, js, logger.Printf)) actedKV.Ready(again.bound(ctx, js, logger.Printf)) errs := make(chan error, 10) - b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} - b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465) + b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the // indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since. b.Log = func(e builds.BuildLogEntry) { @@ -462,12 +461,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro stepReads.expired(ctx, e) return } - if key, ok := versions.MemberBoundOf(e.Key()); ok { - if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member - b.MemberBoundEnded(ctx, key) - } - return - } if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended reads.expiredCall(node) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go deleted file mode 100644 index 63e15b10..00000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-builder/internal/versions" -) - -// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of -// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded). -// No timer. -type memberBounds struct { - bounds func() jetstream.KeyValue - now func() time.Time - logf func(string, ...any) -} - -// Arm is build.MemberBounds' Arm. -func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error { - kv := m.bounds() - if kv == nil { - return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) - } - if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { - return err - } - return nil -} - -// End is build.MemberBounds' End. -func (m *memberBounds) End(ctx context.Context, key string) { - if kv := m.bounds(); kv != nil { - if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - m.logf("the member bound of %s is not deleted: %v", key, err) - } - } -} diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index cddbda4d..0c7ea2c0 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -2,121 +2,11 @@ package build import ( "context" - "fmt" - "maps" - "slices" - "sort" - "strings" - "time" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a -// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.), and -// the builder builds each, once, as a key of its own (builds..) that the checker's line reads. The -// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes -// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as -// the checker's fetch is, and builds the merge's module directory. - -// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk -// commit the head is merged onto. -type via struct { - Ref, Base string - // Members are the group's package members the module is built against by their builds (design 47 G §373, §465). - Members []member -} - -// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's -// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate. -type member struct{ Package, Module, Artifact, Fp string } - -// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it -// carries. A value that is not a candidate is said and left. -func (b *Builder) Candidate(ctx context.Context, key string, value []byte) { - b.init() - c, err := meshcatalog.DecodeCandidatesByGroupValue(value) - if err != nil { - b.logf("candidates.%s holds what is not a candidate, left: %v", key, err) - return - } - b.mu.Lock() - b.cands[key] = c - b.mu.Unlock() - b.considerCandidate(ctx, c) -} - -// ForgetCandidate takes a candidate out (its key was deleted). -func (b *Builder) ForgetCandidate(key string) { - b.init() - b.mu.Lock() - delete(b.cands, key) - b.mu.Unlock() -} - -func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) { - group := map[string]string{} // the fingerprint of each module the group's pull requests change - for _, p := range c.Pulls { - for name, cm := range p.Fps { - if cm.Fp2 != "" && cm.Manifest != "" { - group[name] = cm.Fp2 - } - } - } - for _, p := range c.Pulls { - names := make([]string, 0, len(p.Fps)) - for name := range p.Fps { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - cm := p.Fps[name] - if cm.Fp2 == "" || cm.Manifest == "" { - continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by - } - b.mu.Lock() - trunk, known := b.known[name] - b.mu.Unlock() - if !known || trunk.Path == "" { - b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.Fp2) - continue - } - key, err := builds.BuilderBuildKey(name, cm.Fp2) - if err != nil { - b.logf("candidates.%s: %v", c.Group, err) - continue - } - m := meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: p.Head, Fp2: cm.Fp2, Manifest: cm.Manifest, Path: trunk.Path, - Toolchains: CatalogToolchains(b.agreedFor(ctx, cm.Manifest)), Packages: cm.Packages} // the group's package heads (ADR 0573 §4) - if len(cm.Paths) > 0 { - // The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4). - m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Paths: cm.Paths} - } - // The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's - // repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the - // commit its fingerprint first appeared at, and may be many merges behind the trunk. - base, err := b.trunkHead(ctx, uint64(c.Base), p.Repository) - if err != nil { - b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) - continue - } - members, err := b.membersOf(ctx, name, cm, group) - if err != nil { - b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) - continue - } - b.mu.Lock() - if _, onTrunk := b.trunkFP[key]; !onTrunk { - b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members} - } - b.mu.Unlock() - b.consider(ctx, name, m) - } - } -} - // agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the // candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed // is left out, and the build then waits for it as the trunk's does. @@ -140,192 +30,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]bui } return out } - -// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said. -func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) { - if b.Trunk == nil { - return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository) - } - head, err := b.Trunk(ctx, rev, repository) - if err != nil { - return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err) - } - if head == "" { - return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository) - } - return head, nil -} - -// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package -// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the -// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds, -// is no member: it is fetched as before. -func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) { - if b.Members == nil { - return nil, nil - } - var out []member - for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) { - if len(cm.Packages[pkg]) != 40 { - continue - } - module, artifact, found, err := b.Members(ctx, pkg) - if err != nil { - return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err) - } - if !found || module == name || group[module] == "" { - continue - } - out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]}) - } - return out, nil -} - -// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's -// 60-minute bound). -const MemberBound = 60 * time.Minute - -// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77): -// Arm writes it for key, the dependent's build key; End deletes it. -type MemberBounds interface { - Arm(ctx context.Context, key, why string, bound time.Duration) error - End(ctx context.Context, key string) -} - -// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the -// candidates are looked at again, which fails its step naming the member still not built. -func (b *Builder) MemberBoundEnded(ctx context.Context, key string) { - b.init() - b.mu.Lock() - waiting := b.memberWaits[key] - if waiting { - b.memberExpired[key] = true - } - b.mu.Unlock() - if waiting { - b.TrunkMoved(ctx) - } -} - -// endMemberWait forgets key's wait and deletes its bound. -func (b *Builder) endMemberWait(ctx context.Context, key string) { - b.mu.Lock() - had := b.memberWaits[key] - delete(b.memberWaits, key) - delete(b.memberExpired, key) - b.mu.Unlock() - if had && b.MemberBounds != nil { - b.MemberBounds.End(ctx, key) - } -} - -// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the -// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or -// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the -// builder holds no timer. -func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) { - b.mu.Lock() - v := b.via[key] - b.mu.Unlock() - if len(v.Members) == 0 { - return nil, "", false - } - defer func() { - if !waiting { - b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound - } - }() - var pending []string - for _, mem := range v.Members { - ref := mem.Module + "/" + mem.Artifact - mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp) - if err != nil { - return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false - } - raw, _, found, err := b.Claimer.Store.Get(ctx, mk) - if err != nil { - return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false - } - if !found { - pending = append(pending, ref) - continue - } - cur, err := builds.DecodeBuilderBuild(raw) - if err != nil { - return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false - } - switch cur.State { - case builds.BuildDone: - digest := "" - for _, a := range cur.Artifacts { - if a.Name == mem.Artifact { - digest = a.Digest - } - } - if digest == "" { - return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false - } - got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest}) - case builds.BuildFailed: - return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false - default: - pending = append(pending, ref) - } - } - if len(pending) == 0 { - b.mu.Lock() - b.fetched[key] = got - b.mu.Unlock() - return got, "", false - } - b.mu.Lock() - said := b.memberWaits[key] - expired := b.memberExpired[key] - b.mu.Unlock() - if expired { - return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false - } - if !said { - line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String() - if b.MemberBounds != nil { - // The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written - // would never end, so the step fails at once, saying so. - if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil { - return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false - } - } - b.mu.Lock() - b.memberWaits[key] = true - b.mu.Unlock() - b.logf("%s", line) - if b.Log != nil { - b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"}) - } - } - return nil, "", true -} - -// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's -// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits. -func (b *Builder) memberEnded(ctx context.Context, key string) { - b.mu.Lock() - member := false - for _, v := range b.via { - for _, mem := range v.Members { - if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key { - member = true - } - } - } - b.mu.Unlock() - if member { - b.TrunkMoved(ctx) - } -} - -// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none. -func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild { - b.mu.Lock() - defer b.mu.Unlock() - return b.fetched[key] -} diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go deleted file mode 100644 index a20e2c40..00000000 --- a/modules/mesh-delivery-builder/internal/build/candidates_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package build - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -// trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1) -// is one the trunk does not carry. -func trunkModule(t *testing.T, r *rig) { - t.Helper() - data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), - Fp2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}) - if err != nil { - t.Fatal(err) - } - r.b.Module(context.Background(), "m", data) - r.b.Wait() - r.bm.mu.Lock() - r.bm.calls = nil - r.bm.mu.Unlock() -} - -func candidateValue(t *testing.T, group string, number int, head string) []byte { - t.Helper() - pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: int64(number), Head: head, Modules: []string{"m"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a')}}}} - hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) - if err != nil { - t.Fatal(err) - } - data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) - if err != nil { - t.Fatal(err) - } - return data -} - -const ( - // trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the - // commit its fingerprint first appeared at. - trunkHead = "3333333333333333333333333333333333333333" - headX = "1111111111111111111111111111111111111111" - headY = "2222222222222222222222222222222222222222" -) - -// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer -// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test). -func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c meshcatalog.CandidatesByGroupValue, raw []byte) { - t.Helper() - data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json") - if err != nil { - t.Fatal(err) - } - var w struct { - Source struct { - Rev uint64 `json:"rev"` - Heads map[string]string `json:"heads"` - } `json:"source"` - Candidate json.RawMessage `json:"candidate"` - } - if err := json.Unmarshal(data, &w); err != nil { - t.Fatal(err) - } - if c, err = meshcatalog.DecodeCandidatesByGroupValue(w.Candidate); err != nil { - t.Fatal(err) - } - return w.Source.Rev, w.Source.Heads, c, w.Candidate -} - -func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684) - // The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module - // value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change). - { - rev, heads, c, raw := indexerCandidate(t) - p := c.Pulls[0] - cm := p.Fps["avahi"] - ir := newRig(t, goBundle) - ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} - ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) { - if at != rev { - return "", fmt.Errorf("source revision %d is not heard", at) - } - return heads[repository], nil - } - const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" - value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: valueCommit, Fp2: "src2:" + hex64('e'), - Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: CatalogToolchains(agreedToolchains)}) - if err != nil { - t.Fatal(err) - } - ir.b.Module(context.Background(), "avahi", value) - ir.b.Wait() - ir.bm.mu.Lock() - ir.bm.calls = nil - ir.bm.mu.Unlock() - ir.b.Candidate(context.Background(), "feat%2Fx", raw) - ir.b.Wait() - fetches := ir.bm.callsOf("build_go_fetch") - if len(fetches) != 1 { - t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches)) - } - f := fetches[0].body.(map[string]any) - if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head { - t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev) - } - } - r := newRig(t, goBundle) - trunkModule(t, r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildDone || len(v.Artifacts) != 1 { - t.Fatalf("build = %+v, want the candidate's fingerprint built", v) - } - fetches := r.bm.callsOf("build_go_fetch") - if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4) - t.Fatalf("%d fetches, want one", len(fetches)) - } - f := fetches[0].body.(map[string]any) - if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" { - t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f) - } - if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" { - t.Errorf("build args = %+v", b) - } - // Two heads carrying one fingerprint are one build. - r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY)) - r.b.Wait() - if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { - t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n) - } -} - -func TestACandidateWhoseModuleHasNoTrunkRecordIsNotBuilt(t *testing.T) { - r := newRig(t, goBundle) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - if _, ok := r.st.vals[r.key]; ok { - t.Error("a build key was written for a module the builder knows no directory of") - } -} - -func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { // the trunk's source wins - r := newRig(t, goBundle) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the trunk now carries fp1 - r.b.Wait() - for _, c := range r.bm.callsOf("build_go_fetch") { - if f := c.body.(map[string]any); f["ref"] != nil || f["base"] != nil { - t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f) - } - } - if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { - t.Errorf("build = %+v", v) - } -} - -func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t *testing.T) { // review D4 - for name, set := range map[string]func(*rig){ - "conflict": func(r *rig) { r.bm.conflicts = "go.mod" }, - "no merge": func(r *rig) { r.bm.noMerge = true }, - } { - r := newRig(t, goBundle) - trunkModule(t, r) - set(r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 { - t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build"))) - } - } -} diff --git a/modules/mesh-delivery-builder/internal/build/members_test.go b/modules/mesh-delivery-builder/internal/build/members_test.go deleted file mode 100644 index 7354e477..00000000 --- a/modules/mesh-delivery-builder/internal/build/members_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package build - -import ( - "context" - "encoding/json" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -const memberPackage = "example.org/p" - -var fpP = "src2:" + hex64('2') - -// groupValue is a group's candidate: the module m, built against the head of the package member p, and p. -func groupValue(t *testing.T) []byte { - t.Helper() - pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{ - "m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}}, - "p": {Fp2: fpP, Manifest: mdigest('b')}}}} - hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}}) - if err != nil { - t.Fatal(err) - } - data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) - if err != nil { - t.Fatal(err) - } - return data -} - -// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known. -func memberRig(t *testing.T, known bool) *rig { - t.Helper() - r := newRig(t, goBundle) - lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"} - r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}} - r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) { - if pkg == memberPackage { - return "p", "lib", true, nil - } - return "", "", false, nil - } - trunkModule(t, r) - if known { - data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'), - Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)}) - if err != nil { - t.Fatal(err) - } - r.b.Module(context.Background(), "p", data) - r.b.Wait() - } - return r -} - -func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103 - t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) { - r := memberRig(t, true) - r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { - t.Fatalf("m = %+v, want built", v) - } - var found bool - for _, c := range r.bm.callsOf("build_go_fetch") { - f := c.body.(map[string]any) - bs, _ := f["builds"].([]any) - if len(bs) == 0 { - continue - } - found = true - want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest} - if len(bs) != 1 || !jsonEqual(bs[0], want) { - t.Errorf("builds = %v, want %v", bs, want) - } - pk, _ := f["packages"].(map[string]any) - if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" { - t.Errorf("packages = %v, want the member out of it and the other package kept", pk) - } - } - if !found { - t.Error("no fetch named the member's build") - } - }) - t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) { - r := memberRig(t, false) - var mu sync.Mutex - bounds := &fakeMemberBounds{} - r.b.MemberBounds = bounds - var said []builds.BuildLogEntry - r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() } - r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) - r.b.Wait() - if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 { - t.Fatal("the dependent was claimed or built while its package member is not built") - } - mu.Lock() - if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") { - t.Errorf("said = %+v, want one wait-held naming p/lib", said) - } - mu.Unlock() - if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() { - t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got) - } - r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry - r.b.Wait() - if got := bounds.ended(); len(got) != 1 || got[0] != r.key { - t.Errorf("bounds ended = %v, want the wait's bound deleted", got) - } - if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 { - t.Errorf("m = %+v, want failed naming the member and nothing built", v) - } - }) - t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) { - r := memberRig(t, false) - mk, _ := builds.BuilderBuildKey("p", fpP) - data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"}) - if err != nil { - t.Fatal(err) - } - r.st.put(mk, data) - r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 { - t.Errorf("m = %+v, want failed naming the member and its error", v) - } - }) -} - -// fakeMemberBounds keeps the bounds a builder arms and ends. -type fakeMemberBounds struct { - mu sync.Mutex - arms, ends_ []string -} - -func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error { - f.mu.Lock() - defer f.mu.Unlock() - f.arms = append(f.arms, key+" "+bound.String()) - return nil -} - -func (f *fakeMemberBounds) End(_ context.Context, key string) { - f.mu.Lock() - defer f.mu.Unlock() - f.ends_ = append(f.ends_, key) -} - -func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms } -func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ } - -func jsonEqual(a, b any) bool { - x, _ := json.Marshal(a) - y, _ := json.Marshal(b) - return string(x) == string(y) -} diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index 73b1145e..63731dc6 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -2,45 +2,19 @@ package build import ( "context" - "encoding/json" "strings" "testing" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" ) // R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait // (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4). -func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3 - r := newRig(t, goBundle, archiveA) - trunkModule(t, r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildDone { - t.Fatalf("build = %+v, want done", v) - } - if ok, missing := v.Referenced(); !ok { - t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts) - } - rev := r.st.revs[r.key] - r.bm.mu.Lock() - r.bm.calls = nil - r.bm.mu.Unlock() - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint - r.b.Wait() - if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev { - t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev) - } -} - func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2 r := newRig(t, goBundle) r.bm.noRef["ace/tool"] = true @@ -63,7 +37,7 @@ func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1 } } -func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key +func TestATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a) r := newRig(t, goBundle) r.consider(t, "modules/m") for _, c := range r.bm.callsOf("build_go_build") { @@ -71,15 +45,6 @@ func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's t.Errorf("trunk build args = %+v, want publish", c.body) } } - c := newRig(t, goBundle) - trunkModule(t, c) - c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - c.b.Wait() - for _, call := range c.bm.callsOf("build_go_build") { - if _, ok := call.body.(map[string]any)["publish"]; ok { - t.Errorf("candidate build args = %+v, want no publish", call.body) - } - } } // putBuild writes a builder key's value as the store would hold it. @@ -164,25 +129,10 @@ func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4 var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"} -func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4 - c := newRig(t, tsPackage) - c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - trunkModule(t, c) - c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - c.b.Wait() - v := valueOf(t, c.st, c.key) - if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" { - t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v) - } - for _, call := range c.bm.callsOf("build_node_build") { - if _, ok := call.body.(map[string]any)["publish"]; ok { - t.Errorf("a candidate's package build was asked to publish: %+v", call.body) - } - } - +func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublished(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4 r := newRig(t, tsPackage) r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - v = r.consider(t, "modules/m") + v := r.consider(t, "modules/m") if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" { t.Fatalf("trunk build = %+v, want done naming the published version", v) } @@ -207,50 +157,6 @@ func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B } } -func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6 - r := newRig(t, tsPackage, tsBundle) - r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - trunkModule(t, r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { - t.Fatalf("candidate build = %+v", v) - } - rev := r.st.revs[r.key] - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint - r.b.Wait() - v := valueOf(t, r.st, r.key) - if r.st.revs[r.key] <= rev { - t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev) - } - if ok, missing := v.Referenced(); !ok { - t.Fatalf("build = %+v, missing references %v", v, missing) - } - rev = r.st.revs[r.key] - r.b.tried = map[string]bool{} - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if r.st.revs[r.key] != rev { - t.Error("a published package was built again: once only") - } -} - -func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6 - r := newRig(t, tsPackage) - r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - trunkModule(t, r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - for _, n := range []string{"ace", "novox"} { - r.bm.fails[n+"/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" - } - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) { - t.Errorf("build = %+v, want failed opening %q", v, builds.VersionTaken) - } -} - // sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref. func sdkModule(t *testing.T, r *rig, ref string) []byte { t.Helper() @@ -365,30 +271,6 @@ func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: } } -func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths - r := newRig(t, tsPackage) - r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - trunkModule(t, r) - pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}} - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) - data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) - if err != nil { - t.Fatal(err) - } - r.b.Candidate(context.Background(), "feat%2Fx", data) - r.b.Wait() - f := r.bm.callsOf("build_node_fetch") - if len(f) == 0 { - t.Fatal("no fetch") - } - req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any)) - // The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one). - if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead { - t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead) - } -} - // 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that // answered a digest alone (a build module from before R4e). func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e37f226f..66c5ae51 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -77,14 +77,9 @@ type Builder struct { // Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done // key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing. Stored func(ctx context.Context, references []string) (gone []string, err error) - // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a - // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. - Trunk func(ctx context.Context, rev uint64, repository string) (string, error) // Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G - // §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a - // member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none. - Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error) - MemberBounds MemberBounds + // §373, §465): the part a resolved package names in a build's dependencies. Nil names none. + Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error) // Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. @@ -132,8 +127,6 @@ type Builder struct { again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built plans map[string]planned // by manifest digest stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go) - cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go) - via map[string]via // by build key: a candidate's fetch (candidates.go) // outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them. outOfStep map[string][]string trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint @@ -141,10 +134,6 @@ type Builder struct { // published are the package versions `@` a done key of this bucket names as a package's reference: a key // waiting for one is built once it is here (ADR 0500 clause 6). published map[string]bool - // memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired - // those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go). - memberWaits, memberExpired map[string]bool - fetched map[string][]sdkstep.FetchBuild // held are the done keys' references the store answered held, by ` `, asked once per process: a // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). held map[string]bool @@ -181,13 +170,12 @@ func (b *Builder) init() { limit = 256 } b.sem = make(chan struct{}, limit) - b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} + b.trunkFP = map[string]bool{} b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{} b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{} b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands, b.events = map[string]*handWatch{}, map[string]bool{} - b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{} b.builds = map[string]context.CancelCauseFunc{} }) } @@ -265,7 +253,6 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. b.trunkFP[bk] = true b.origin[bk] = o b.edges[bk] = e - delete(b.via, bk) } b.mu.Unlock() if superseded != nil { @@ -472,23 +459,6 @@ func (b *Builder) Reconsider(ctx context.Context) { for _, k := range sortedKeys(known) { b.consider(ctx, k, known[k]) } - b.TrunkMoved(ctx) -} - -// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the -// revision it was previewed onto had no trunk head to be merged onto. -func (b *Builder) TrunkMoved(ctx context.Context) { - b.init() - b.mu.Lock() - cands := make([]meshcatalog.CandidatesByGroupValue, 0, len(b.cands)) - for _, c := range b.cands { - cands = append(cands, c) - } - b.mu.Unlock() - sort.Slice(cands, func(i, j int) bool { return cands[i].Group < cands[j].Group }) - for _, c := range cands { - b.considerCandidate(ctx, c) - } } func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string { @@ -630,13 +600,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } } - memberFail := "" - if !trunk && planErr == nil && !nothing { - var waiting bool - if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting { - return // a package member of the group is not built (design 47 G §373): its end looks at this key again - } - } // Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371). reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { @@ -654,7 +617,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } b.handClaimed(key, cl.Rev) - b.logBuild(LogStarted(module, m, b.refOf(module, m.Fp2), b.Node, b.now())) + b.logBuild(LogStarted(module, m, "", b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -686,8 +649,6 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB // Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key // fails with why, and is taken again at the next look. result = failed("%s", numbering.Refused) - } else if memberFail != "" && cl.Build.State != builds.BuildDone { - result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373) } else if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). @@ -734,8 +695,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB if b.Ended != nil { b.Ended(ctx, cl.Key) } - b.memberEnded(ctx, cl.Key) - ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result) + ended := LogEnded(module, m, "", b.Node, b.now(), result) ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26) b.logBuild(ended) if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 { @@ -1074,15 +1034,6 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error { // key taken over at start waits for the ask the key names and puts none. func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) { req := sdkstep.Fetch{Repository: repository, Commit: m.Commit} - if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { - b.mu.Lock() - v, candidate := b.via[key] - b.mu.Unlock() - if candidate { - req.Ref, req.Base = v.Ref, v.Base - req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465) - } - } if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 { // A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from // the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e). @@ -1092,17 +1043,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } sort.Strings(req.Paths) } - candidate := req.Base != "" seat, _ := b.Seats.OfModule(l.module) language := seat.AskLanguage() kind := sdkstep.AskDelivery - if candidate { - kind = sdkstep.AskCheck - } job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain), - // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, - // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). - Publish: !candidate} + Publish: true} for _, t := range l.tasks { a := sdkstep.AskArtifact{Artifact: t.Artifact} if t.Kind == catalogue.ArtifactUpstream { @@ -1197,7 +1142,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } l.digests[t.Artifact] = a.Digest l.resolved[t.Artifact] = a.Resolved - if a.Reference != "" && (stored(t.Kind) || !candidate) { + if a.Reference != "" { l.references[t.Artifact] = a.Reference } if a.Reference != "" && b.Answered != nil { @@ -1236,17 +1181,6 @@ func (b *Builder) logBuild(e builds.BuildLogEntry) { b.Log(e) } -// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's. -func (b *Builder) refOf(module, fp string) string { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return "" - } - b.mu.Lock() - defer b.mu.Unlock() - return b.via[key].Ref -} - // LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the // source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder // that asked it and the time. diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 0e8c77b0..7451bff1 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -187,7 +187,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { r.b = &Builder{ Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second}, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf, - Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil }, } var err error if r.key, err = builds.BuilderBuildKey("m", fp1); err != nil { @@ -437,31 +436,6 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a } } -func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision - r := newRig(t, imageArt) - r.puller = "ace" - trunkModule(t, r) - // Design 47 G §373: a candidate's build is numbered under no tag, so it numbers nothing. - nums := &fakeNumbers{} - r.b.Numbers = nums - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - if nums.numbered != 0 { - t.Errorf("a candidate's build was numbered %d times, want none", nums.numbered) - } - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { - t.Fatalf("build = %+v, want done with no reference", v) - } - calls := r.bm.callsOf("build_image_build") - if len(calls) != 1 { - t.Fatalf("%d image builds, want 1", len(calls)) - } - if _, has := calls[0].body.(map[string]any)["publish"]; has { - t.Errorf("a candidate's image call carries publish: %+v", calls[0].body) - } -} - func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim r := newRig(t, imageArt) r.puller = "ace" @@ -793,81 +767,6 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/ } } -func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5 - r := newRig(t, imageArt) - r.puller = "ace" - trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildDone || v.Artifacts[0].Reference != "" { - t.Fatalf("candidate build = %+v, want done with no reference", v) - } - if n := len(r.bm.callsOf("build_image_build")); n != 1 { - t.Fatalf("%d image builds after the candidate, want 1", n) - } - // The pull request is merged: the trunk's record names the candidate's fingerprint. - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - v = valueOf(t, r.st, r.key) - if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" { - t.Fatalf("trunk build = %+v, want done with a reference", v) - } - calls := r.bm.callsOf("build_image_build") - published := 0 - for _, c := range calls { - if c.body.(map[string]any)["publish"] == true { - published++ - } - } - if len(calls) != 2 || published != 1 { - t.Errorf("%d image builds, %d with publish, want 2 and exactly 1", len(calls), published) - } - // Written once more only: looking again builds nothing. - rev := r.st.revs[r.key] - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_image_build")) != 2 { - t.Error("a published key was built again") - } -} - -func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5 - r := newRig(t, imageArt) - r.puller = "ace" - trunkModule(t, r) - r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { - t.Fatalf("candidate build = %+v", v) - } - r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused" - // Issue 1748: a run of the ask that ended on a transport fault is said in the log and on the key, and is no end. - inner, log := r.asks.Answer, &logged{} - r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { - r.asks.Write(buildstep.Step{ID: a.ID, Node: "g14", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Reason: "lookup registry: i/o timeout"}) - return inner(a) - } - r.b.Logf = log.logf - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint - r.b.Wait() - r.asks.Answer, r.b.Logf = inner, t.Logf - if !log.has("run 1 of", "on g14 ended on a transport fault", "lookup registry: i/o timeout") { - t.Errorf("a run given again was not said: %v", log.lines) - } - v := valueOf(t, r.st, r.key) - if v.State != builds.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") { - t.Fatalf("build = %+v, want failed naming the publish and its reason", v) - } - // The failed key is retaken as any failed key is, and ends published once the registry answers. - delete(r.bm.fails, "ace/build_image_build") - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Artifacts[0].Reference == "" { - t.Errorf("build = %+v, want done with a reference", v) - } -} - // goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to // drop from their stream (Dropper). type goneAsks struct { diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 506d12ac..e996f757 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -73,11 +73,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc } m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path, Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages} - b.mu.Lock() - if _, onTrunk := b.trunkFP[key]; !onTrunk { - b.via[key] = via{} - } - b.mu.Unlock() b.consider(ctx, entry.Module, m) select { case log := <-ended: diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index 44176615..82fe99dd 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -38,16 +38,8 @@ const ( oldFormBoundPrefix = "old-form." unputBoundPrefix = "unput." stepsBoundPrefix = "steps." - memberBoundPrefix = "member." ) -// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): -// `member.`. Its end while the member is not built fails the dependent's step naming the member. -func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey } - -// MemberBoundOf is the build key a member wait's bound names, and whether key is one. -func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) } - // UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, // novox/hq ADR 0668 §3): `unput.`. Its end while the key still stands so, with no look holding it, is the // finding call-unanswered about the key.