mesh-delivery-indexer: a release re-pins or is not acknowledged; pins enter the version line's fingerprint

A re-pin that left the fingerprint as it was left the build made
against the older pin as the one the mesh runs. The version line's
fingerprint now takes the pins, so a re-pin moves the importer, which
becomes a mover built against the release. An accepted series' older
version no longer stands over a newer release. A re-pin that fails
returns its error before the heard artifacts are forgotten, and a
package-published heard again for a released version re-pins, so the
redelivery recovers it. The release path and the seed's pins carry
tests that fail without them (review round 2).

Builds: design 47 H H-A-165, H-A-167, H-A-161
This commit is contained in:
2026-11-20 17:46:53 +01:00
parent 037cb9555e
commit fe51379ee2
7 changed files with 105 additions and 26 deletions
@@ -481,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
// NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own
// git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree).
const NamedTreePrefix = "named:"
// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no
// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin <package> <version>" of each pin,
// joined by a newline. A package kept with no version (none released) is no pin.
func WithPins(fp string, pins sdkstep.PackageVersions) string {
var lines []string
for p, v := range pins {
if v != "" {
lines = append(lines, "pin "+p+" "+v)
}
}
if fp == "" || len(lines) == 0 {
return fp
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n")))
return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:])
}
@@ -17,6 +17,7 @@ import (
type form struct {
fp, none string
versions sdkstep.PackageVersions
line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167)
}
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
@@ -71,7 +72,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
if fp == "" {
return form{none: none}
}
return form{fp: fp, versions: versions}
return form{fp: fp, versions: versions, line: true}
}
// refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod
@@ -272,6 +272,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
}
// mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167).
ix.Versions = newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" {
t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages)
}
}
// Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that
@@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
fp := "src2:" + strings.Repeat("a", 64)
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{},
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}},
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"},
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161)
Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
@@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
t.Errorf("the image artifact is %+v", a)
}
in := got[0]
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" {
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" {
t.Errorf("the steps input is %+v", in)
}
if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned {
t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages)
}
if len(w.Held()) != 0 {
t.Fatal("a call unanswered for no time wrote its finding")
}
@@ -291,17 +291,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok {
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok {
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
}
default:
f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf)
r.fp, r.none = f.fp, f.none
if r.fp != "" {
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
r.fp, r.pkgs = pinnedOf(f, held, released)
r.tcs = toolchainsUsed(toolchains)
}
}
r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161)
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
!importsOf(r.pkgs, repository) || only != nil && !only[module]) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the
@@ -497,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool {
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
// module has none until they agree.
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) {
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) {
if k.held.Fp2 == "" {
return "", nil, false
}
@@ -514,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil)
return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2
if f.none != "" || f.fp == "" {
return "", nil, false
}
fp, pkgs := pinnedOf(f, held, released)
return fp, pkgs, fp == k.held.Fp2
}
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
@@ -17,6 +17,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -783,15 +784,33 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
mods.put("mesh-tools", mustEncode(t, stale))
tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "")
ix.Versions = newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
// mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the
// pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A
// re-pin that fails is no release acknowledged: the package-published delivered again re-pins.
ix.Versions, ix.Heard = newMem(), newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`))
// mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165).
if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 {
t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err)
revisions := ix.Revisions
ix.Revisions = newMemObjects(1 << 20)
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}})
if err != nil {
t.Fatal(err)
}
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) {
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages)
if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"}
mods.keysErr = errors.New("the bus did not answer")
if err := ix.HearPackagePublished(context.Background(), release); err == nil {
t.Fatal("a release whose re-pin failed was acknowledged")
}
mods.keysErr = nil
if err := ix.HearPackagePublished(context.Background(), release); err != nil {
t.Fatal(err)
}
ix.Revisions = revisions
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) {
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 {
t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2)
@@ -127,8 +127,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
return err
} else if found && !isFloor(raw) {
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact)
return nil
// A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else.
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact)
return ix.repin(ctx, p.Module, p.Version)
}
entry, err := ix.entryOf(ctx, p.Module)
if err != nil {
@@ -238,11 +239,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
}
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
}
if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165)
if n, err := ix.Repin(ctx, p.Module); err != nil {
ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err)
} else if n > 0 {
ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version)
if !late {
// The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not
// acknowledged, so the package-published is delivered again (H-A-167).
if err := ix.repin(ctx, p.Module, p.Version); err != nil {
return err
}
}
if err := ix.trimVersions(ctx, p.Module); err != nil {
@@ -399,11 +400,14 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.
}
out := sdkstep.PackageVersions{}
for p, v := range versions {
switch key := fingerprint.HolderOf(p, held); {
case sdkstep.PackageSemver.MatchString(v):
rel := released[moduleOfKey(fingerprint.HolderOf(p, held))]
series := sdkstep.PackageSemver.MatchString(v)
_, plain := versionParts(strings.TrimPrefix(v, "v"))
switch {
case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0):
out[p] = "v" + rel // never an older series' version over a newer release (H-A-167)
case series:
out[p] = v
case released[moduleOfKey(key)] != "":
out[p] = "v" + released[moduleOfKey(key)]
default:
out[p] = ""
}
@@ -411,6 +415,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.
return out
}
// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the
// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is.
func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, held, released)
if !f.line {
return f.fp, pins
}
return fingerprint.WithPins(f.fp, pins), pins
}
// moduleOfKey is the module a modules key names; "" for none.
func moduleOfKey(key string) string {
if key == "" {
@@ -437,3 +452,15 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions {
}
return out
}
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
if n > 0 {
ix.logf("versions: %d modules re-pinned to %s %s", n, module, version)
}
return nil
}