diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go index 3d030bb1..ca501971 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go @@ -79,6 +79,12 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf if kept { logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published) } + if kept && h.Published { + // A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard. + if err := ix.Recompute(ctx); err != nil { + logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err) + } + } _ = m.Ack() return true } diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index 5fdc9c37..70421e2e 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -1,6 +1,7 @@ package index import ( + "slices" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -29,9 +30,12 @@ type form struct { // another repository its go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), and the // content of each mesh Go module of its own repository its go.mod replaces by a directory (fingerprint.VersionLine), // so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. +// - **An accepted series** (DLV-I6): published and pins come from acceptedSeries, so a mesh Go module of another +// repository whose series the module accepts enters, in either form, by its series' newest published version and the +// digests of that version's commit. // // It sets kept.ImportForm when it writes the import form. -func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, logf func(string, ...any)) form { +func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, pins delivery.PackageVersions, logf func(string, ...any)) form { in := inputsFrom(module, *kept, toolchains) if len(kept.Imports) > 0 { r := fingerprint.Resolve(repository, kept.Imports, published) @@ -51,7 +55,17 @@ func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains ma logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", ")) } } - versions, unpublished := fingerprint.HeldVersionsOf(fingerprint.OtherPackages(kept.Packages, repository), held) + others := fingerprint.OtherPackages(kept.Packages, repository) + versions, unpublished := fingerprint.HeldVersionsOf(others, held) + for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6) + if slices.Contains(others, p) { + if versions == nil { + versions = delivery.PackageVersions{} + } + versions[p] = v + unpublished = slices.DeleteFunc(unpublished, func(u string) bool { return u == p }) + } + } if len(unpublished) > 0 && logf != nil { logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) } diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index 81aa6261..b00dd7be 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "slices" + "strings" "git.novox.be/novox/mesh-delivery/sdk/buildlog" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -105,18 +106,35 @@ func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e } published = ix.publishedAll(ctx) } + // A pin that is an accepted series' version (DLV-I6) is taken by the contents its series keeps for it. + byVersion := map[string]delivery.PublishedPackages{} + for _, s := range ix.SeriesAll(ctx) { + if s.GoModule != "" && s.Newest != nil && s.Newest.Contents != nil { + byVersion[s.GoModule+"@v"+s.Newest.Version] = *s.Newest.Contents + } + } type at struct{ repository, commit string } reread := map[at][]string{} + var asVersion map[string]delivery.PublishedPackages for _, imp := range e.Imports { r := fingerprint.Resolve(cur.Repository, []string{imp}, published) for mod := range r.Requires { p := published[mod] commit := p.Commit - switch { + switch pin := cur.Packages[mod]; { case delivery.SameRepository(p.Repository, cur.Repository) && e.Commit != "": commit = e.Commit - case cur.Packages[mod] != "": - commit = cur.Packages[mod] + case strings.HasPrefix(pin, "v"): + if c, ok := byVersion[mod+"@"+pin]; ok { + if asVersion == nil { + asVersion = map[string]delivery.PublishedPackages{} + } + c.Commit, asVersion[mod] = pin, c + } else { + ix.logf("%s: the version %s of %s it was pinned to is no series' newest any more, so its imports are read at the latest published commit", e.Module, pin, mod) + } + case pin != "": + commit = pin } if commit != p.Commit { k := at{p.Repository, commit} @@ -134,5 +152,6 @@ func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e } maps.Copy(out, got) } + maps.Copy(out, asVersion) return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index ef512ed4..0453fb2c 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -49,6 +49,7 @@ func (m *missingDir) Error() string { return "no directory " + m.dir } func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T) { t.Run("a build of an older fingerprint keeps no imports", buildOfAnOlderFingerprintKeepsNoImports) t.Run("a change of a sibling directory alone moves its importer", aSiblingDirectoryAloneMovesItsImporter) + t.Run("an accepted series' newest published version pins it, by that version's contents", anAcceptedSeriesPinsItsNewest) ctx := context.Background() ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") @@ -209,3 +210,61 @@ func setPackages(t *testing.T, mods *memStore, module string, packages []string) } mods.put(module, raw) } + +// DLV-I6 (ADR 0606 §4, ADR 0609 §1; design 32 *A package, its series, and the series its users accept*): avahi accepts +// mesh-sdk/go's default series. Once a seat publishes 0.1.15 from c3 (announce changed), avahi is pinned to v0.1.15 and +// fingerprinted by c3's digests; a publish of another series and a later trunk merge that changes announce back (c2) do +// not move it; before any publish it kept the commit form. +func anAcceptedSeriesPinsItsNewest(t *testing.T) { + ctx := context.Background() + ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + ix.Packages, ix.Files = newMem(), forge{c1: sdkFiles("a1", "b1"), c2: sdkFiles("a1", "b2"), c3: sdkFiles("a2", "b2")}.files + setPackages(t, mods, "avahi", []string{meshSDK}) + sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"}) + if err != nil { + t.Fatal(err) + } + mods.put("mesh-sdk", sdkModule) + step := func(what string, act func() error) delivery.Module { + t.Helper() + if err := act(); err != nil { + t.Fatalf("%s: %v", what, err) + } + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "avahi") + return m + } + trunk := func(commit string) func() error { + return func() error { + held, _ := ix.heldModules(ctx) + return ix.publish(ctx, "novox/mesh-sdk", commit, held, false) + } + } + hear := func(version, commit string) func() error { + return func() error { + _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: version, + Commit: commit, GoModule: meshSDK, Published: true}) + return err + } + } + before := step("trunk at c1", trunk(c1)) + if before.Packages[meshSDK] != c1 { + t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages) + } + step("the build reports its imports", func() error { + return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce")) + }) + pinned := step("0.1.15 published from c3", hear("0.1.15", c3)) + if pinned.FP2 == before.FP2 || pinned.Packages[meshSDK] != "v0.1.15" { + t.Fatalf("after 0.1.15: fp2 %q (was %q), versions %v; want moved by c3's announce, pinned to v0.1.15", pinned.FP2, before.FP2, pinned.Packages) + } + if m := step("0.2.0 published from c1", hear("0.2.0", c1)); m.FP2 != pinned.FP2 || m.Packages[meshSDK] != "v0.1.15" { + t.Errorf("a publish of a series avahi does not accept moved it: %q %v", m.FP2, m.Packages) + } + if m := step("trunk at c2", trunk(c2)); m.FP2 != pinned.FP2 || m.Packages[meshSDK] != "v0.1.15" { + t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.FP2, m.Packages) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index bb367fb6..c07abd9e 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -301,12 +301,32 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive // The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk // computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed. published := ix.candidatePublished(ctx, members, held) + // A package of a repository outside the group by the series the candidate's manifest accepts (DLV-I6); a member's by the + // group's head. + var pins delivery.PackageVersions + if _, man, err := ix.manifestRaw(ctx, pm.Manifest); err == nil { + series := ix.SeriesAll(ctx) + for key, s := range series { + if repo, _, _ := delivery.PackageRepository(s.GoModule); members[repo] != "" { + delete(series, key) + } + } + published, pins = acceptedSeries(series, man, pv.Repository, published) + } if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 { - f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, nil) + f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, pins, nil) fp, none, versions = f.fp, f.none, f.versions } else { inputs.Imports, inputs.ImportForm = nil, "" versions = candidatePackages(fingerprint.OtherPackages(inputs.Packages, pv.Repository), held, members) + for p, v := range pins { + if _, named := versions[p]; named || slices.Contains(inputs.Packages, p) { + if versions == nil { + versions = delivery.PackageVersions{} + } + versions[p] = v + } + } in := inputsFrom(m, inputs, toolchains) in.Packages = fingerprint.VersionLine(pv.Repository, inputs.Packages, versions, published) fp, none = fingerprint.Of(in) diff --git a/modules/mesh-delivery-indexer/internal/index/series.go b/modules/mesh-delivery-indexer/internal/index/series.go index 84dc9bdf..ec628a32 100644 --- a/modules/mesh-delivery-indexer/internal/index/series.go +++ b/modules/mesh-delivery-indexer/internal/index/series.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "fmt" + "maps" + "slices" "strings" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -42,6 +45,10 @@ type SeriesVersion struct { Commit string `json:"commit,omitempty"` Digest string `json:"digest,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` + // Contents are, for a series' newest of a Go package, its Go module's requirements' and packages' digests read from the + // forge's listing at Commit (DLV-I6): what a user accepting the series is fingerprinted against. Nil when the forge + // could not list it (said), and on a numbered version. + Contents *delivery.PublishedPackages `json:"contents,omitempty"` } // SeriesKey is the key of a package's series in the packages state. @@ -77,6 +84,21 @@ func (ix *Indexer) HearVersion(ctx context.Context, h HeardVersion) (bool, error return false, nil } v := &SeriesVersion{Version: strings.TrimPrefix(h.Version, "v"), Commit: h.Commit, Digest: h.Digest, Fingerprint: h.Fingerprint} + numbered := *v + if h.Published && h.GoModule != "" && h.Commit != "" && ix.Files != nil { + // The digests of the published version's commit, read before the write (DLV-I6): a user accepting the series is + // fingerprinted against what this version holds, not against the latest merge. + repo, _, _ := delivery.PackageRepository(h.GoModule) + got, err := ix.packagesAt(ctx, repo, h.Commit, []string{h.GoModule}, nil) + switch p, ok := got[h.GoModule]; { + case err != nil: + ix.logf("%s %s: its contents at %s are not read, so its users keep the commit form until its next publish: %v", h.Package, h.Version, h.Commit, err) + case !ok: + ix.logf("%s %s: the forge's listing at %s holds no Go module %s, so its users keep the commit form", h.Package, h.Version, h.Commit, h.GoModule) + default: + v.Contents = &p + } + } newer := func(held *SeriesVersion) bool { return held == nil || CompareVersions("v"+held.Version, "v"+v.Version) < 0 } @@ -91,7 +113,7 @@ func (ix *Indexer) HearVersion(ctx context.Context, h HeardVersion) (bool, error } kept = false if newer(s.Numbered) { - s.Numbered, kept = v, true + s.Numbered, kept = &numbered, true } if h.Published && newer(s.Newest) { s.Newest, kept = v, true @@ -140,3 +162,35 @@ func (ix *Indexer) SeriesAll(ctx context.Context) map[string]SeriesPackage { } return out } + +// acceptedSeries is the packages state as a module of repository whose manifest is m is fingerprinted against (DLV-I6; +// ADR 0606 §4, ADR 0609 §1; design 32 *A package, its series, and the series its users accept*): each mesh Go module of +// another repository whose package's series the module accepts (Manifest.AcceptedSeries) and a seat published, by that +// series' newest version, its digests read at that version's commit, and pinned to the version itself (Go's form, +// `v0.1.15`). A series with no published version yet, or whose contents the forge did not list, leaves the module's +// latest published commit (the commit form, milestone § 2 item 4). A module of the same repository takes its sibling by +// path, never by a series. With nothing to pin, published is answered as it is. +func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repository string, published map[string]delivery.PublishedPackages) (map[string]delivery.PublishedPackages, delivery.PackageVersions) { + var out map[string]delivery.PublishedPackages + var pins delivery.PackageVersions + for _, key := range slices.Sorted(maps.Keys(series)) { + s := series[key] + if s.GoModule == "" || s.Newest == nil || s.Newest.Contents == nil || m.AcceptedSeries(s.Package) != s.Series { + continue + } + if repo, _, ok := delivery.PackageRepository(s.GoModule); !ok || delivery.SameRepository(repo, repository) { + continue + } + if out == nil { + out, pins = maps.Clone(published), delivery.PackageVersions{} + } + p := *s.Newest.Contents + version := "v" + s.Newest.Version + p.Commit = version + out[s.GoModule], pins[s.GoModule] = p, version + } + if out == nil { + return published, nil + } + return out, pins +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 8218c5f7..b19fe0c2 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -201,6 +201,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli } tools := ix.agreed() published := ix.publishedAll(ctx) + series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) type result struct { fp, none string tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint @@ -257,6 +258,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli toolchains[artifact] = t.tc } var r result + pub, pins := acceptedSeries(series, k.manifest, k.cur.Repository, published) switch { case k.contexts: r.none = None2Context @@ -264,11 +266,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, published); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: - f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, published, ix.logf) + f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions @@ -464,7 +466,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages) (string, delivery.PackageVersions, bool) { +func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, pins delivery.PackageVersions) (string, delivery.PackageVersions, bool) { if k.held.FP2 == "" { return "", nil, false } @@ -480,7 +482,7 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma } toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } - f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, nil) + f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, nil) return f.fp, f.versions, f.none == "" && f.fp == k.held.FP2 }