mesh-delivery-indexer, mesh-delivery-builder: a package is pinned by its released version, never by a commit
The indexer filled a module's packages with the commit of a module value it holds, which Go turned into a pseudo-version of whatever that tree was (mesh-delivery-sdk at an 11-19 commit, nodeengine pre-H1). A pin is now the newest released version of the module holding the package (the versions state version-released says), an accepted series' version stands, and a package with none released is kept with no version: its kind-1 edge stands and the builder asks no pin for it. A commit of another repository moves a module only through a package of it the module imports (A-476a), now that pins name no commit. Builds: design 47 H H-A-161, G §370, H §746, H-A-157 (kind-1 edges)
This commit is contained in:
@@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
|
||||
for _, fb := range req.Builds {
|
||||
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
|
||||
}
|
||||
|
||||
@@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
||||
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil // no node has pulled it yet
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
|
||||
// released, design 47 H-A-161) is no pin.
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk})
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+5
-1
@@ -144,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge})
|
||||
heard.next(t, "main", 10)
|
||||
heard.acked(t, "main")
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 })
|
||||
// mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161).
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool {
|
||||
pin, kept := m.Packages[sdkGo]
|
||||
return m.Fp2 != "" && kept && pin == ""
|
||||
})
|
||||
plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" })
|
||||
if _, ok := plain.Packages[sdkGo]; ok {
|
||||
t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages)
|
||||
|
||||
@@ -423,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys
|
||||
// commit or no directory, is in missing, sorted; versions is nil when packages is empty.
|
||||
func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) {
|
||||
for _, pkg := range packages {
|
||||
repository, dir, ok := trunk.PackageRepository(pkg)
|
||||
commit, depth := "", -1
|
||||
if ok {
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == meshcatalogclient.RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
commit, depth = m.Commit, d
|
||||
}
|
||||
}
|
||||
}
|
||||
if commit == "" {
|
||||
key := HolderOf(pkg, modules)
|
||||
if key == "" {
|
||||
missing = append(missing, pkg)
|
||||
continue
|
||||
}
|
||||
if versions == nil {
|
||||
versions = sdkstep.PackageVersions{}
|
||||
}
|
||||
versions[pkg] = commit
|
||||
versions[pkg] = modules[key].Commit
|
||||
}
|
||||
slices.Sort(missing)
|
||||
return versions, missing
|
||||
}
|
||||
|
||||
// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the
|
||||
// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none.
|
||||
func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string {
|
||||
repository, dir, ok := trunk.PackageRepository(pkg)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
holder, depth := "", -1
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == meshcatalogclient.RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
holder, depth = key, d
|
||||
}
|
||||
}
|
||||
return holder
|
||||
}
|
||||
|
||||
// digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it
|
||||
// (its packages are sorted by import path).
|
||||
func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
|
||||
|
||||
@@ -118,8 +118,8 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _ := moduleOf(t, mods, "avahi")
|
||||
if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
|
||||
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3",
|
||||
if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" {
|
||||
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)",
|
||||
moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages)
|
||||
}
|
||||
}
|
||||
@@ -256,8 +256,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
before := step("trunk at c1", trunk(c1))
|
||||
if before.Packages[meshSDK] != c1 {
|
||||
t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages)
|
||||
if pin, kept := before.Packages[meshSDK]; !kept || pin != "" {
|
||||
t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages)
|
||||
}
|
||||
step("the build reports its imports", func() error {
|
||||
return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce"))
|
||||
|
||||
@@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
|
||||
seat := seatOf[a.Ref]
|
||||
sa := SeedArtifact{Name: name, Seat: seat}
|
||||
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
|
||||
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}}
|
||||
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}}
|
||||
// The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause
|
||||
// 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest).
|
||||
if paths := namedPaths(e, name); len(paths) > 0 {
|
||||
|
||||
@@ -215,7 +215,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
ix.logf("no fingerprint is written by this pass: %v", err)
|
||||
return fmt.Errorf("no fingerprint is written: %w", err)
|
||||
}
|
||||
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
|
||||
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
|
||||
released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161)
|
||||
type result struct {
|
||||
fp, none string
|
||||
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
|
||||
@@ -297,8 +298,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
|
||||
}
|
||||
}
|
||||
r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161)
|
||||
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
|
||||
!importsOf(r.pkgs, repository) {
|
||||
// A module registered for another repository is moved by this repository's commit only when it imports a package the
|
||||
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
|
||||
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
|
||||
@@ -577,3 +579,14 @@ func (ix *Indexer) fillPaths(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of
|
||||
// repository moves a module of another only through a package of it the module imports (A-476a).
|
||||
func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
|
||||
for p := range pkgs {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -757,7 +757,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2)
|
||||
}
|
||||
tools, _ := moduleOf(t, mods, "mesh-tools")
|
||||
sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue {
|
||||
sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue {
|
||||
t.Helper()
|
||||
f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1,
|
||||
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}}
|
||||
@@ -765,9 +765,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
|
||||
// which imports nothing it moved.
|
||||
// which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none
|
||||
// released it has no pin, the package kept for its kind-1 edge (H-A-161).
|
||||
a, _ := moduleOf(t, mods, "avahi")
|
||||
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
|
||||
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) {
|
||||
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil {
|
||||
@@ -781,8 +782,11 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
|
||||
mods.put("mesh-tools", mustEncode(t, stale))
|
||||
tools = stale
|
||||
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
|
||||
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "")
|
||||
ix.Versions = newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`))
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20")
|
||||
if second.Fp2 == first.Fp2 {
|
||||
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
|
||||
}
|
||||
|
||||
@@ -10,10 +10,14 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
|
||||
@@ -352,3 +356,77 @@ func isFloor(raw []byte) bool {
|
||||
var v meshcatalog.VersionEntry
|
||||
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
|
||||
}
|
||||
|
||||
// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module;
|
||||
// nil when the indexer keeps no such state or it cannot be listed, said.
|
||||
func (ix *Indexer) releasedOf(ctx context.Context) map[string]string {
|
||||
if ix.Versions == nil {
|
||||
return nil
|
||||
}
|
||||
all, err := ix.Versions.Keys(ctx)
|
||||
if err != nil {
|
||||
ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, k := range all {
|
||||
m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
|
||||
if _, parts := versionParts(v); !ok || !parts {
|
||||
continue
|
||||
}
|
||||
if cur, has := out[m]; !has || compareVersions(v, cur) > 0 {
|
||||
out[m] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
|
||||
// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned
|
||||
// to that module's newest released version, as Go names it (v<version>); a version already in that form (an accepted
|
||||
// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge
|
||||
// stands (H §746) and no build is made against a commit. nil for none.
|
||||
func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions {
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := sdkstep.PackageVersions{}
|
||||
for p, v := range versions {
|
||||
switch key := fingerprint.HolderOf(p, held); {
|
||||
case sdkstep.PackageSemver.MatchString(v):
|
||||
out[p] = v
|
||||
case released[moduleOfKey(key)] != "":
|
||||
out[p] = "v" + released[moduleOfKey(key)]
|
||||
default:
|
||||
out[p] = ""
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleOfKey is the module a modules key names; "" for none.
|
||||
func moduleOfKey(key string) string {
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
if m, err := meshcatalogclient.ParseModulesKey(key); err == nil {
|
||||
return m
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with
|
||||
// none (no released version, H-A-161) is no pin. nil for none.
|
||||
func PinsOf(packages map[string]string) sdkstep.PackageVersions {
|
||||
var out sdkstep.PackageVersions
|
||||
for p, v := range packages {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = sdkstep.PackageVersions{}
|
||||
}
|
||||
out[p] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user