diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 7a329c8c..9eaed460 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) + maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161) for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 41f4d44f..a13d4c37 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet - // Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with. - sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} + // Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none + // released, design 47 H-A-161) is no pin. + sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"} data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", - Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk}) + Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 603c875e..e2f46323 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -144,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) heard.next(t, "main", 10) heard.acked(t, "main") - untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 }) + // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161). + untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { + pin, kept := m.Packages[sdkGo] + return m.Fp2 != "" && kept && pin == "" + }) plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" }) if _, ok := plain.Packages[sdkGo]; ok { t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index 4a326972..c5a4e5cb 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -423,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys // commit or no directory, is in missing, sorted; versions is nil when packages is empty. func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) { for _, pkg := range packages { - repository, dir, ok := trunk.PackageRepository(pkg) - commit, depth := "", -1 - if ok { - for _, key := range slices.Sorted(maps.Keys(modules)) { - m := modules[key] - if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { - continue - } - at := strings.Trim(m.Path, "/") - switch { - case at == "": - continue // a value that names no directory says nothing of where the package lies - case at == meshcatalogclient.RootPath: - at = "" - case dir != at && !strings.HasPrefix(dir, at+"/"): - continue - } - if d := len(at); d > depth { - commit, depth = m.Commit, d - } - } - } - if commit == "" { + key := HolderOf(pkg, modules) + if key == "" { missing = append(missing, pkg) continue } if versions == nil { versions = sdkstep.PackageVersions{} } - versions[pkg] = commit + versions[pkg] = modules[key].Commit } slices.Sort(missing) return versions, missing } +// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the +// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none. +func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string { + repository, dir, ok := trunk.PackageRepository(pkg) + if !ok { + return "" + } + holder, depth := "", -1 + for _, key := range slices.Sorted(maps.Keys(modules)) { + m := modules[key] + if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" { + continue + } + at := strings.Trim(m.Path, "/") + switch { + case at == "": + continue // a value that names no directory says nothing of where the package lies + case at == meshcatalogclient.RootPath: + at = "" + case dir != at && !strings.HasPrefix(dir, at+"/"): + continue + } + if d := len(at); d > depth { + holder, depth = key, d + } + } + return holder +} + // digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it // (its packages are sorted by import path). func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 0579ca8a..222cf39f 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -118,8 +118,8 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T t.Fatal(err) } moved, _ := moduleOf(t, mods, "avahi") - if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 { - t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3", + if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" { + t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)", moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages) } } @@ -256,8 +256,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { } } before := step("trunk at c1", trunk(c1)) - if before.Packages[meshSDK] != c1 { - t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages) + if pin, kept := before.Packages[meshSDK]; !kept || pin != "" { + t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages) } step("the build reports its imports", func() error { return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce")) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 1aed0f99..9667902c 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. seat := seatOf[a.Ref] sa := SeedArtifact{Name: name, Seat: seat} in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, - Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}} + Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}} // The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause // 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest). if paths := namedPaths(e, name); len(paths) > 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ae5b21e4..25975a4a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -215,7 +215,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc ix.logf("no fingerprint is written by this pass: %v", err) return fmt.Errorf("no fingerprint is written: %w", err) } - series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) + released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161) type result struct { fp, none string tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint @@ -297,8 +298,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } + r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && - maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { + !importsOf(r.pkgs, repository) { // A module registered for another repository is moved by this repository's commit only when it imports a package the // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} @@ -577,3 +579,14 @@ func (ix *Indexer) fillPaths(ctx context.Context) error { } return nil } + +// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of +// repository moves a module of another only through a package of it the module imports (A-476a). +func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { + for p := range pkgs { + if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 6357bc82..ec8c3e5e 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -757,7 +757,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2) } tools, _ := moduleOf(t, mods, "mesh-tools") - sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue { + sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue { t.Helper() f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} @@ -765,9 +765,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { t.Fatal(err) } // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, - // which imports nothing it moved. + // which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none + // released it has no pin, the package kept for its kind-1 edge (H-A-161). a, _ := moduleOf(t, mods, "avahi") - if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { + if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) { t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil { @@ -781,8 +782,11 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { stale.Fp2 = "src2:" + strings.Repeat("5", 64) mods.put("mesh-tools", mustEncode(t, stale)) tools = stale - first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") - second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") + first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) + second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20") if second.Fp2 == first.Fp2 { t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 574a6fb5..1a14fb21 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -10,10 +10,14 @@ import ( "strconv" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-indexer/internal/fingerprint" ) // The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's @@ -352,3 +356,77 @@ func isFloor(raw []byte) bool { var v meshcatalog.VersionEntry return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 } + +// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module; +// nil when the indexer keeps no such state or it cannot be listed, said. +func (ix *Indexer) releasedOf(ctx context.Context) map[string]string { + if ix.Versions == nil { + return nil + } + all, err := ix.Versions.Keys(ctx) + if err != nil { + ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err) + return nil + } + out := map[string]string{} + for _, k := range all { + m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k) + if _, parts := versionParts(v); !ok || !parts { + continue + } + if cur, has := out[m]; !has || compareVersions(v, cur) > 0 { + out[m] = v + } + } + return out +} + +// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its +// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned +// to that module's newest released version, as Go names it (v); a version already in that form (an accepted +// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge +// stands (H §746) and no build is made against a commit. nil for none. +func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions { + if len(versions) == 0 { + return nil + } + out := sdkstep.PackageVersions{} + for p, v := range versions { + switch key := fingerprint.HolderOf(p, held); { + case sdkstep.PackageSemver.MatchString(v): + out[p] = v + case released[moduleOfKey(key)] != "": + out[p] = "v" + released[moduleOfKey(key)] + default: + out[p] = "" + } + } + return out +} + +// moduleOfKey is the module a modules key names; "" for none. +func moduleOfKey(key string) string { + if key == "" { + return "" + } + if m, err := meshcatalogclient.ParseModulesKey(key); err == nil { + return m + } + return key +} + +// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with +// none (no released version, H-A-161) is no pin. nil for none. +func PinsOf(packages map[string]string) sdkstep.PackageVersions { + var out sdkstep.PackageVersions + for p, v := range packages { + if v == "" { + continue + } + if out == nil { + out = sdkstep.PackageVersions{} + } + out[p] = v + } + return out +}