mesh-delivery-indexer, mesh-delivery-builder: a package is pinned by its released version, never by a commit

The indexer filled a module's packages with the commit of a module value
it holds, which Go turned into a pseudo-version of whatever that tree
was (mesh-delivery-sdk at an 11-19 commit, nodeengine pre-H1). A pin is
now the newest released version of the module holding the package (the
versions state version-released says), an accepted series' version
stands, and a package with none released is kept with no version: its
kind-1 edge stands and the builder asks no pin for it. A commit of
another repository moves a module only through a package of it the
module imports (A-476a), now that pins name no commit.

Builds: design 47 H H-A-161, G §370, H §746, H-A-157 (kind-1 edges)
This commit is contained in:
2026-11-20 17:46:53 +01:00
parent e04321acae
commit c43e7a5d1c
9 changed files with 150 additions and 40 deletions
@@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names. // them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages) ask.Packages = maps.Clone(m.Packages)
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
for _, fb := range req.Builds { for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
} }
@@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle) r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet r.asks.Answer = nil // no node has pulled it yet
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with. // Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} // released, design 47 H-A-161) is no pin.
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk}) Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -144,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge})
heard.next(t, "main", 10) heard.next(t, "main", 10)
heard.acked(t, "main") heard.acked(t, "main")
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 }) // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161).
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool {
pin, kept := m.Packages[sdkGo]
return m.Fp2 != "" && kept && pin == ""
})
plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" }) plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" })
if _, ok := plain.Packages[sdkGo]; ok { if _, ok := plain.Packages[sdkGo]; ok {
t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages) t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages)
@@ -423,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys
// commit or no directory, is in missing, sorted; versions is nil when packages is empty. // commit or no directory, is in missing, sorted; versions is nil when packages is empty.
func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) { func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) {
for _, pkg := range packages { for _, pkg := range packages {
repository, dir, ok := trunk.PackageRepository(pkg) key := HolderOf(pkg, modules)
commit, depth := "", -1 if key == "" {
if ok {
for _, key := range slices.Sorted(maps.Keys(modules)) {
m := modules[key]
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
continue
}
at := strings.Trim(m.Path, "/")
switch {
case at == "":
continue // a value that names no directory says nothing of where the package lies
case at == meshcatalogclient.RootPath:
at = ""
case dir != at && !strings.HasPrefix(dir, at+"/"):
continue
}
if d := len(at); d > depth {
commit, depth = m.Commit, d
}
}
}
if commit == "" {
missing = append(missing, pkg) missing = append(missing, pkg)
continue continue
} }
if versions == nil { if versions == nil {
versions = sdkstep.PackageVersions{} versions = sdkstep.PackageVersions{}
} }
versions[pkg] = commit versions[pkg] = modules[key].Commit
} }
slices.Sort(missing) slices.Sort(missing)
return versions, missing return versions, missing
} }
// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the
// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none.
func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string {
repository, dir, ok := trunk.PackageRepository(pkg)
if !ok {
return ""
}
holder, depth := "", -1
for _, key := range slices.Sorted(maps.Keys(modules)) {
m := modules[key]
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
continue
}
at := strings.Trim(m.Path, "/")
switch {
case at == "":
continue // a value that names no directory says nothing of where the package lies
case at == meshcatalogclient.RootPath:
at = ""
case dir != at && !strings.HasPrefix(dir, at+"/"):
continue
}
if d := len(at); d > depth {
holder, depth = key, d
}
}
return holder
}
// digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it // digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it
// (its packages are sorted by import path). // (its packages are sorted by import path).
func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
@@ -118,8 +118,8 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
t.Fatal(err) t.Fatal(err)
} }
moved, _ := moduleOf(t, mods, "avahi") moved, _ := moduleOf(t, mods, "avahi")
if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 { if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" {
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3", t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)",
moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages) moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages)
} }
} }
@@ -256,8 +256,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
} }
} }
before := step("trunk at c1", trunk(c1)) before := step("trunk at c1", trunk(c1))
if before.Packages[meshSDK] != c1 { if pin, kept := before.Packages[meshSDK]; !kept || pin != "" {
t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages) t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages)
} }
step("the build reports its imports", func() error { step("the build reports its imports", func() error {
return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce")) return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce"))
@@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
seat := seatOf[a.Ref] seat := seatOf[a.Ref]
sa := SeedArtifact{Name: name, Seat: seat} sa := SeedArtifact{Name: name, Seat: seat}
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}} Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}}
// The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause // The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause
// 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest). // 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest).
if paths := namedPaths(e, name); len(paths) > 0 { if paths := namedPaths(e, name); len(paths) > 0 {
@@ -215,7 +215,8 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
ix.logf("no fingerprint is written by this pass: %v", err) ix.logf("no fingerprint is written by this pass: %v", err)
return fmt.Errorf("no fingerprint is written: %w", err) return fmt.Errorf("no fingerprint is written: %w", err)
} }
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6) series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161)
type result struct { type result struct {
fp, none string fp, none string
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
@@ -297,8 +298,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
} }
} }
r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161)
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) { !importsOf(r.pkgs, repository) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the // A module registered for another repository is moved by this repository's commit only when it imports a package the
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's. // commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)} r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
@@ -577,3 +579,14 @@ func (ix *Indexer) fillPaths(ctx context.Context) error {
} }
return nil return nil
} }
// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of
// repository moves a module of another only through a package of it the module imports (A-476a).
func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
for p := range pkgs {
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) {
return true
}
}
return false
}
@@ -757,7 +757,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2) t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2)
} }
tools, _ := moduleOf(t, mods, "mesh-tools") tools, _ := moduleOf(t, mods, "mesh-tools")
sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue { sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue {
t.Helper() t.Helper()
f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1, f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1,
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}}
@@ -765,9 +765,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools, // A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
// which imports nothing it moved. // which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none
// released it has no pin, the package kept for its kind-1 edge (H-A-161).
a, _ := moduleOf(t, mods, "avahi") a, _ := moduleOf(t, mods, "avahi")
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) { if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages) t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
} }
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil { if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil {
@@ -781,8 +782,11 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
stale.Fp2 = "src2:" + strings.Repeat("5", 64) stale.Fp2 = "src2:" + strings.Repeat("5", 64)
mods.put("mesh-tools", mustEncode(t, stale)) mods.put("mesh-tools", mustEncode(t, stale))
tools = stale tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1") first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2") ix.Versions = newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`))
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20")
if second.Fp2 == first.Fp2 { if second.Fp2 == first.Fp2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2) t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
} }
@@ -10,10 +10,14 @@ import (
"strconv" "strconv"
"strings" "strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go" stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/internal/fingerprint"
) )
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's // The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
@@ -352,3 +356,77 @@ func isFloor(raw []byte) bool {
var v meshcatalog.VersionEntry var v meshcatalog.VersionEntry
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
} }
// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module;
// nil when the indexer keeps no such state or it cannot be listed, said.
func (ix *Indexer) releasedOf(ctx context.Context) map[string]string {
if ix.Versions == nil {
return nil
}
all, err := ix.Versions.Keys(ctx)
if err != nil {
ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err)
return nil
}
out := map[string]string{}
for _, k := range all {
m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if _, parts := versionParts(v); !ok || !parts {
continue
}
if cur, has := out[m]; !has || compareVersions(v, cur) > 0 {
out[m] = v
}
}
return out
}
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned
// to that module's newest released version, as Go names it (v<version>); a version already in that form (an accepted
// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge
// stands (H §746) and no build is made against a commit. nil for none.
func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions {
if len(versions) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for p, v := range versions {
switch key := fingerprint.HolderOf(p, held); {
case sdkstep.PackageSemver.MatchString(v):
out[p] = v
case released[moduleOfKey(key)] != "":
out[p] = "v" + released[moduleOfKey(key)]
default:
out[p] = ""
}
}
return out
}
// moduleOfKey is the module a modules key names; "" for none.
func moduleOfKey(key string) string {
if key == "" {
return ""
}
if m, err := meshcatalogclient.ParseModulesKey(key); err == nil {
return m
}
return key
}
// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with
// none (no released version, H-A-161) is no pin. nil for none.
func PinsOf(packages map[string]string) sdkstep.PackageVersions {
var out sdkstep.PackageVersions
for p, v := range packages {
if v == "" {
continue
}
if out == nil {
out = sdkstep.PackageVersions{}
}
out[p] = v
}
return out
}