diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index cdc26dbd..a5167640 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -228,6 +228,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } b.PackageWait = time.Duration(packageWait) * time.Minute + // Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194). + b.PublishedVersions = publishedVersionsVia(calls.Call) waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error { return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body) @@ -242,9 +244,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p) } b.Reserved = reserved - // A release heard, or the newest the builder's record keeps, pins a package an entry names with none (H-A-165). - reserved.heardReleased = b.VersionReleased - b.NewestReleased = reserved.r.NewestReleased // The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it. reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps) // A state bound later numbers the builds looked at without it. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 18b19181..8923151c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -44,15 +44,12 @@ type reservations struct { // built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none. built func(ctx context.Context, module, version string) error released func(ctx context.Context, module, version string) - // heardReleased is told each version-released taken in (design 47 H H-A-165): a step ask waiting for a package of the - // module wakes. Nil tells nobody. - heardReleased func(module, version string) - filled atomic.Bool - bounds func() jetstream.KeyValue - found *findings - bound time.Duration - wake func(context.Context) - logf func(string, ...any) + filled atomic.Bool + bounds func() jetstream.KeyValue + found *findings + bound time.Duration + wake func(context.Context) + logf func(string, ...any) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller // say says package-published on the seat mesh-builds; nil says none. @@ -255,9 +252,6 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { if rv.released != nil { rv.released(ctx, ev.Module, ev.Version) } - if rv.heardReleased != nil { - rv.heardReleased(ev.Module, ev.Version) - } rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) default: return false diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index cd9b6791..11075cbd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -134,15 +134,10 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") { t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err) } - var heardReleases []string // H-A-165: a release heard wakes a step ask waiting for a package of the module - rv.heardReleased = func(m, v string) { heardReleases = append(heardReleases, m+" "+v) } released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)} if !rv.heard(ctx, released) || !released.acked { t.Fatalf("version-released was not taken in (acked %v)", released.acked) } - if strings.Join(heardReleases, ", ") != "m 1.4.3" { - t.Errorf("the release heard was told %v, want m 1.4.3 (H-A-165)", heardReleases) - } if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" { t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go index 51283e17..3bcc405d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go @@ -105,6 +105,10 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool { if h.res != nil && se.event == versions.HeardPublished { h.res.published(ctx, o) } + if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil { + // A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192). + h.b.PackagePublished(o.Name, o.Version) + } h.ack(m) if len(wake) > 0 { h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index 1dfae047..fbd625ac 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" @@ -96,6 +97,28 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e return out, nil } +// publishedVersionsVia asks the Go package registry seat's verb versions for one package path (design 47 H H-A-194): its +// versions, newest first. +func publishedVersionsVia(call toolCaller) func(context.Context, string) ([]string, error) { + return func(_ context.Context, path string) ([]string, error) { + raw, err := call("seat:"+gopackageregistry.Seat+"."+gopackageregistry.VerbVersions, gopackageregistry.VersionsInput{Name: path}) + if err != nil { + return nil, err + } + var a gopackageregistry.VersionsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", gopackageregistry.Seat, err) + } + var out []string + for _, v := range a.Versions { + if v.Version != "" { + out = append(out, v.Version) + } + } + return out, nil + } +} + // floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb // versions, the seat git's tags and the requirements the catalog half's entries name (A-119). A call that fails is that source // silent. diff --git a/modules/mesh-delivery-builder/internal/build/entry_test.go b/modules/mesh-delivery-builder/internal/build/entry_test.go index 5ccbf04f..f2e4f2f1 100644 --- a/modules/mesh-delivery-builder/internal/build/entry_test.go +++ b/modules/mesh-delivery-builder/internal/build/entry_test.go @@ -13,6 +13,7 @@ import ( func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) { e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f", ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"}, + Artifacts: []meshcatalog.Artifact{{Ref: "netcheck:lib", Kind: "package", Language: "go"}}, Inputs: &meshcatalog.EntryInputs{Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}}, Paths: map[string][]string{"lib": {"a/b", "c"}}}} got := ModuleOf(e) @@ -30,4 +31,11 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) { if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" { t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare) } + // A module with an artifact that names no paths (a Go program beside its packages) is built from its directory whole + // (ADR 0499 clauses 1 and 4): its fetch names no paths, so the program's go.mod and main package stay in the tree. + mixed := e + mixed.Artifacts = append([]meshcatalog.Artifact{{Ref: "netcheck:program", Kind: "bundle", Language: "go"}}, e.Artifacts...) + if got := ModuleOf(mixed); got.Inputs2 == nil || len(got.Inputs2.Paths) != 0 { + t.Errorf("ModuleOf of a module with a program naming no paths = %+v, want bases and no named paths", got.Inputs2) + } } diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 6739c40b..2a052bbc 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -5,11 +5,12 @@ import ( "fmt" "maps" "slices" + "strconv" + "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) // **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step @@ -104,13 +105,16 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou } } -// **A package with no pin waits for its release** (design 47 H H-A-165, H-A-161): an entry keeps a mesh package none of -// whose versions is released with an empty version. Its step ask waits, as for its revision, for the release of the module -// that holds the package (the catalog's version-released, VersionReleased), up to PackageWait (the setting -// package-wait-minutes), then builds with that release as the package's pin (v); past the bound the step fails, -// naming the package, and nothing is built. +// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry +// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its +// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions, +// H-A-194), then waits for the registry seat's published of that package path in that series (the +// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to +// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a +// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and +// nothing is built. -// DefaultPackageWait is the bound of a step ask's wait for a package's release, its setting package-wait-minutes' default. +// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default. const DefaultPackageWait = 10 * time.Minute func (b *Builder) packageWait() time.Duration { @@ -120,91 +124,152 @@ func (b *Builder) packageWait() time.Duration { return DefaultPackageWait } -// VersionReleased takes the catalog's version-released of module in: a step ask waiting for a package of it wakes. -func (b *Builder) VersionReleased(module, version string) { +// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the +// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package +// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing. +func (b *Builder) PackagePublished(path, version string) { + v := strings.TrimPrefix(version, "v") + series, ok := seriesOfVersion(v) + if path == "" || !ok { + return + } b.init() b.mu.Lock() - b.releases[module] = version + in := b.publishedPins[path] + if in == nil { + in = map[string]string{} + b.publishedPins[path] = in + } + if old := in[series]; old == "" || compareVersions(v, old) > 0 { + in[series] = v + } b.mu.Unlock() b.pictureMoved() } -// releaseOf is the pin a package is built against: the release of the module holding it, as Go names it; "" for none yet. -func (b *Builder) releaseOf(ctx context.Context, pkg string) string { - if b.Members == nil { - return "" - } - module, _, found, err := b.Members(ctx, pkg) - if err != nil || !found { - return "" - } +// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none. +func (b *Builder) publishedPin(path, series string) string { b.mu.Lock() - v := b.releases[module] + v := b.publishedPins[path][series] b.mu.Unlock() - if v == "" && b.NewestReleased != nil { - if kept, ok, err := b.NewestReleased(ctx, module); err == nil && ok { - v = kept - } - } if v == "" { return "" } return "v" + v } -// awaitPins fills each package entry names with no pin from its release, waiting for one; no outcome once every package has -// a pin, failed past the bound, could not run at the builder's stop. -func (b *Builder) awaitPins(ctx context.Context, entry *meshcatalog.CatalogEntry) (outcome, said string) { +// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest +// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and +// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops. +func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) { var unpinned []string - for p, v := range entry.Packages { + for p, v := range packages { if v == "" { unpinned = append(unpinned, p) } } if len(unpinned) == 0 { - return "", "" + return packages, "", "" } slices.Sort(unpinned) + if b.Manifests == nil { + return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0]) + } + man, err := b.Manifests.Manifest(ctx, digest) + if err != nil { + return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err) + } + // What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it. + for _, p := range unpinned { + if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" { + continue + } + vs, err := b.PublishedVersions(ctx, p) + if err != nil { + b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err) + continue + } + for _, v := range vs { + b.PackagePublished(p, v) + } + } bound := b.packageWait() wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8) defer stop() - pins := map[string]string{} told := false for { b.mu.Lock() moved := b.moved b.mu.Unlock() - missing := "" + pins := map[string]string{} + missing, missingSeries := "", "" for _, p := range unpinned { - if pins[p] == "" { - pins[p] = b.releaseOf(ctx, p) - } + series := man.AcceptedSeries(p) + pins[p] = b.publishedPin(p, series) if pins[p] == "" && missing == "" { - missing = p + missing, missingSeries = p, series } } if missing == "" { - packages := maps.Clone(entry.Packages) - maps.Copy(packages, pins) - entry.Packages = packages + out := maps.Clone(packages) + maps.Copy(out, pins) if told { - b.logf("%s at %s: every package it names has a released version: built with %v", entry.Module, entry.Fp, pins) + b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins) } - return "", "" + return out, "", "" } - notReleased := fmt.Sprintf("package %s has no released version", missing) + notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries) if !told { - b.logf("%s at %s: %s yet: the ask waits for its release, at most %s", entry.Module, entry.Fp, notReleased, bound) + b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound) told = true } select { case <-moved: case <-wait.Done(): if ctx.Err() != nil { - return meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was" + return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was" } - b.logf("%s at %s: %s within %s: the step fails, nothing is built", entry.Module, entry.Fp, notReleased, bound) - return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound) + b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound) + return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound) } } } + +// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form. +func seriesOfVersion(v string) (string, bool) { + if _, ok := versionParts(v); !ok { + return "", false + } + major, rest, _ := strings.Cut(v, ".") + minor, _, _ := strings.Cut(rest, ".") + return major + "." + minor, true +} + +// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form. +func versionParts(v string) ([3]int, bool) { + var out [3]int + parts := strings.Split(v, ".") + if len(parts) != 3 { + return out, false + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 { + return out, false + } + out[i] = n + } + return out, true +} + +// compareVersions orders two MAJOR.MINOR.PATCH versions numerically. +func compareVersions(a, b string) int { + x, _ := versionParts(a) + y, _ := versionParts(b) + for i := range x { + if x[i] != y[i] { + return x[i] - y[i] + } + } + return 0 +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 9eaed460..7f76e535 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -113,11 +113,13 @@ type Builder struct { // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration - // PackageWait bounds a step ask's wait for the release of a package its entry names with no pin, its setting - // package-wait-minutes (H-A-165); DefaultPackageWait when 0. NewestReleased reads the newest version the builder's - // record of releases keeps for a module; nil reads none, and only a release heard (VersionReleased) gives a pin. - PackageWait time.Duration - NewestReleased func(ctx context.Context, module string) (version string, found bool, err error) + // PackageWait bounds a build's wait for the publish of a package its entry names with no pin, its setting + // package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished) + // gives a pin (H-A-192). + PackageWait time.Duration + // PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first + // (H-A-194): read once before a build waits for that package. Nil reads none. + PublishedVersions func(ctx context.Context, path string) ([]string, error) // Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine // answered when last asked, and counted, every node the assignments names, present or away, a node with no statement // stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in @@ -162,8 +164,9 @@ type Builder struct { builds map[string]context.CancelCauseFunc // moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake. moved chan struct{} - // releases are the versions heard released, by module (VersionReleased): a package with no pin waits for one. - releases map[string]string + // publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished): + // a package with no pin waits for one (H-A-192). + publishedPins map[string]map[string]string } type planned struct { @@ -194,7 +197,7 @@ func (b *Builder) init() { b.hands, b.events = map[string]*handWatch{}, map[string]bool{} b.builds = map[string]context.CancelCauseFunc{} b.moved = make(chan struct{}) - b.releases = map[string]string{} + b.publishedPins = map[string]map[string]string{} }) } @@ -285,7 +288,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. // ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository, // directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is -// built against, its packages and needs, and the named paths of its artifacts. +// built against, its packages and needs, and the named paths of its artifacts when every artifact names them. func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None, Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs} @@ -294,7 +297,10 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { } if in := e.Inputs; in != nil { m.Toolchains, m.Problems = in.Toolchains, in.Problems - if len(in.Paths) > 0 { + // Only a module whose every artifact names its paths is built from those alone (ADR 0499 clauses 1, 2 and 4): an + // artifact without paths, such as a Go program, keeps its module's directory as its build source, so a module + // with one names no paths and its fetch keeps the directory whole. + if len(in.Paths) > 0 && everyArtifactNamesPaths(e) { if m.Inputs2 == nil { m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{} } @@ -309,6 +315,20 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { return m } +// everyArtifactNamesPaths is whether the entry names artifacts and the named paths of each, by its reference or its name. +func everyArtifactNamesPaths(e meshcatalog.CatalogEntry) bool { + if len(e.Artifacts) == 0 || e.Inputs == nil { + return false + } + for _, a := range e.Artifacts { + _, name, _ := strings.Cut(a.Ref, ":") + if len(e.Inputs.Paths[a.Ref]) == 0 && len(e.Inputs.Paths[name]) == 0 { + return false + } + } + return true +} + // originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build // of the trunk's value was before branches were watched. func (b *Builder) originOf(key string) Origin { @@ -1112,9 +1132,15 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job} if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 { // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes - // them into the workspace, whatever the repository's go.mod names. - ask.Packages = maps.Clone(m.Packages) - maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161) + // them into the workspace, whatever the repository's go.mod names. A package the entry names with no pin waits for + // its publish and is built with the version that publish carries; past the bound the lane fails naming it, and no + // ask is put (H-A-192, H-A-165). + pinned, outcome, said := b.awaitPublished(ctx, module, m.Fp2, m.Manifest, m.Packages) + if outcome != "" { + l.err = errors.New(said) + return + } + ask.Packages = maps.Clone(pinned) for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index a13d4c37..b611511d 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -333,8 +333,10 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet // Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none - // released, design 47 H-A-161) is no pin. - sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"} + // published, design 47 H-A-161) is pinned at the version the registry seat's published carries (H-A-192), never + // asked without it. + r.b.PackagePublished("git.novox.be/novox/mesh-contracts/go/seats/x", "v0.1.7") + sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.7"} data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}}) diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index af0c9f9d..a61187ed 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -63,10 +63,12 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) { return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit) } - // A package the entry names with no pin is built against its release, waited for (H-A-165). - if outcome, said := b.awaitPins(ctx, &entry); outcome != "" { + // A package the entry names with no pin is built against its publish, waited for (H-A-192, H-A-165). + pinned, outcome, said := b.awaitPublished(ctx, entry.Module, entry.Fp, meshcatalog.BareManifestDigest(entry.ManifestDigest), entry.Packages) + if outcome != "" { return outcome, said } + entry.Packages = pinned key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 91854a2a..6a777cf2 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -3,9 +3,10 @@ package build import ( "context" "errors" + "fmt" "reflect" + "slices" "strings" - "sync" "sync/atomic" "testing" "time" @@ -121,76 +122,155 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk) t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease) t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound) + t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage) + t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait) } -// unpinnedRig is a rig whose step entry m names the package x with no version, built by the module contracts. -func unpinnedRig(t *testing.T) (*rig, *fakeReservations) { +// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it +// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish. +func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) { var held atomic.Int64 - r, res, entry := revisionRig(t, &held, make(chan struct{}, 1)) + r, _, entry := revisionRig(t, &held, make(chan struct{}, 1)) r.b.Picture = nil // no revision to wait for here - entry.Packages = map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/x": "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"} + entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"} r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil } - r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) { - return "contracts", "contracts:x", pkg == "git.novox.be/novox/mesh-contracts/go/seats/x", nil + waiting := make(chan struct{}, 1) + r.b.Logf = func(format string, args ...any) { + t.Logf(format, args...) + if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") { + select { + case waiting <- struct{}{}: + default: + } + } } - return r, res + return r, entry, waiting } -func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { - r, _ := unpinnedRig(t) - var once sync.Once - looked := make(chan struct{}) - r.b.NewestReleased = func(context.Context, string) (string, bool, error) { // the record holds none yet - once.Do(func() { close(looked) }) - return "", false, nil +const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x" + +// awaitWaiting fails the test unless the build started its wait before ended said anything. +func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) { + t.Helper() + select { + case <-waiting: + case end := <-ended: + t.Fatalf("the build ended before the package was published: %+v", end) + case <-time.After(5 * time.Second): + t.Fatal("the build never waited for the package's publish") } +} + +// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in +// another series and the module's release number pin nothing. +func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { + r, _, waiting := unpinnedRig(t) + r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin + r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin ended := make(chan meshbuilds.StepEnded, 1) go func() { ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) }() - select { - case <-looked: - case end := <-ended: - t.Fatalf("the ask ended before the package was released: %+v, asks %+v", end, r.asks.Asks()) - case <-time.After(5 * time.Second): - t.Fatal("the ask never looked for the package's release") - } + awaitWaiting(t, waiting, ended) if _, ok := r.st.vals[r.key]; ok { - t.Fatal("the build was made before the package had a release") + t.Fatal("the build was made before the package was published") } - r.b.VersionReleased("contracts", "0.1.3") + r.b.PackagePublished(unpinnedX, "v0.1.3") var end meshbuilds.StepEnded select { case end = <-ended: case <-time.After(5 * time.Second): - t.Fatal("the release did not wake the ask") + t.Fatal("the publish did not wake the ask") } r.b.Wait() asks := r.asks.Asks() if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || - !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{"git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { + !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks) } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { - r, _ := unpinnedRig(t) + r, _, _ := unpinnedRig(t) r.b.PackageWait = time.Nanosecond end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) r.b.Wait() if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built || - !strings.Contains(end.Said, "package git.novox.be/novox/mesh-contracts/go/seats/x has no released version") { + !strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") { t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built) } - // A release the builder's own record keeps gives the pin at once. - r.b.NewestReleased = func(_ context.Context, module string) (string, bool, error) { - return "0.1.2", module == "contracts", nil - } + // A publish heard before gives the pin at once. + r.b.PackagePublished(unpinnedX, "v0.1.2") if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") { - t.Errorf("a release the record keeps did not pin the package: %+v", end) + t.Errorf("a publish heard did not pin the package: %+v", end) + } +} + +// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series +// pins the package with no event heard; one only in another series leaves the build waiting for a publish. +func theSeatsVersionsPinBeforeTheWait(t *testing.T) { + r, _, _ := unpinnedRig(t) + r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin + var read []string + r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) { + read = append(read, path) + return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first + } + end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + r.b.Wait() + asks := r.asks.Asks() + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) { + t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read) + } + r2, _, waiting := unpinnedRig(t) + r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil } + ended := make(chan meshbuilds.StepEnded, 1) + go func() { + ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + }() + awaitWaiting(t, waiting, ended) + r2.b.PackagePublished(unpinnedX, "0.1.10") + if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed { + t.Errorf("end = %+v, want passed once x was published in its series", end) + } + r2.b.Wait() +} + +// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the +// key past the bound naming the package. +func aLaneWaitsForAnUnpinnedPackage(t *testing.T) { + r, entry, waiting := unpinnedRig(t) + done := make(chan struct{}) + go func() { + r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry) + r.b.Wait() + close(done) + }() + awaitWaiting(t, waiting, done) + if asks := r.asks.Asks(); len(asks) != 0 { + t.Fatalf("a build was asked without the package's pin: %+v", asks) + } + r.b.PackagePublished(unpinnedX, "0.1.5") + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the publish did not wake the lane") + } + if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" { + t.Errorf("asks %+v; want one asked with x at v0.1.5", asks) + } + // Past its bound the key fails naming the package, and nothing is asked. + r2, entry2, _ := unpinnedRig(t) + r2.b.PackageWait = time.Nanosecond + r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2) + r2.b.Wait() + if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") { + t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key]) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 0527980e..0bd46450 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -295,23 +295,6 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) return false, nil } -// NewestReleased is the newest version the builder's own record of releases keeps for module (design 47 H H-A-165): a -// package with no pin is built against it; false when the record holds none. -func (r *Reserver) NewestReleased(ctx context.Context, module string) (string, bool, error) { - if r.Releases == nil { - return "", false, nil - } - raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module)) - if err != nil || !found { - return "", false, err - } - var rec ReleasedRecord - if json.Unmarshal(raw, &rec) != nil || len(rec.Kept) == 0 { - return "", false, nil - } - return rec.Kept[len(rec.Kept)-1].Version, true, nil -} - // keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept. func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error { n := r.ReleasedKept diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 925f49fd..3b68a10b 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -201,13 +201,11 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { t.Errorf("reservations left: %v", all) } // One key per module holds the last releases, the oldest dropped (A-456b). - newest := "" for _, fp := range []string{"src2:b1", "src2:b2"} { n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp}) if err != nil { t.Fatal(err) } - newest = n.Version if _, err := r.Released(ctx, "m", n.Version); err != nil { t.Fatal(err) } @@ -221,13 +219,6 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok { t.Error("the newest release is not kept") } - // H-A-165: the newest version the record keeps is the pin of a package with none. - if v, ok, err := r.NewestReleased(ctx, "m"); !ok || err != nil || v != newest { - t.Errorf("newest released of m = %q, %v (%v), want %s", v, ok, err, newest) - } - if _, ok, _ := r.NewestReleased(ctx, "other"); ok { - t.Error("a module never released has a newest release") - } } // TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces