mesh-delivery-indexer: a release re-pins or is not acknowledged; pins enter the version line's fingerprint

A re-pin that left the fingerprint as it was left the build made
against the older pin as the one the mesh runs. The version line's
fingerprint now takes the pins, so a re-pin moves the importer, which
becomes a mover built against the release. An accepted series' older
version no longer stands over a newer release. A re-pin that fails
returns its error before the heard artifacts are forgotten, and a
package-published heard again for a released version re-pins, so the
redelivery recovers it. The release path and the seed's pins carry
tests that fail without them (review round 2).

Builds: design 47 H H-A-165, H-A-167, H-A-161
This commit is contained in:
2026-11-20 17:46:53 +01:00
parent 037cb9555e
commit fe51379ee2
7 changed files with 105 additions and 26 deletions
@@ -481,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
// NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own // NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own
// git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree). // git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree).
const NamedTreePrefix = "named:" const NamedTreePrefix = "named:"
// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no
// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin <package> <version>" of each pin,
// joined by a newline. A package kept with no version (none released) is no pin.
func WithPins(fp string, pins sdkstep.PackageVersions) string {
var lines []string
for p, v := range pins {
if v != "" {
lines = append(lines, "pin "+p+" "+v)
}
}
if fp == "" || len(lines) == 0 {
return fp
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n")))
return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:])
}
@@ -17,6 +17,7 @@ import (
type form struct { type form struct {
fp, none string fp, none string
versions sdkstep.PackageVersions versions sdkstep.PackageVersions
line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167)
} }
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
@@ -71,7 +72,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
if fp == "" { if fp == "" {
return form{none: none} return form{none: none}
} }
return form{fp: fp, versions: versions} return form{fp: fp, versions: versions, line: true}
} }
// refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod
@@ -272,6 +272,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" { if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages) t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
} }
// mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167).
ix.Versions = newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" {
t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages)
}
} }
// Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that // Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that
@@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
fp := "src2:" + strings.Repeat("a", 64) fp := "src2:" + strings.Repeat("a", 64)
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{},
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}},
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161)
Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil { if err != nil {
@@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
t.Errorf("the image artifact is %+v", a) t.Errorf("the image artifact is %+v", a)
} }
in := got[0] in := got[0]
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" {
t.Errorf("the steps input is %+v", in) t.Errorf("the steps input is %+v", in)
} }
if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned {
t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages)
}
if len(w.Held()) != 0 { if len(w.Held()) != 0 {
t.Fatal("a call unanswered for no time wrote its finding") t.Fatal("a call unanswered for no time wrote its finding")
} }
@@ -291,17 +291,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.none = strings.Join(missing, "; ") r.none = strings.Join(missing, "; ")
case noToolchain != "": case noToolchain != "":
r.none = noToolchain r.none = noToolchain
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok { if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok {
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
} }
default: default:
f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf)
r.fp, r.none = f.fp, f.none r.fp, r.none = f.fp, f.none
if r.fp != "" { if r.fp != "" {
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
r.fp, r.pkgs = pinnedOf(f, held, released)
r.tcs = toolchainsUsed(toolchains)
} }
} }
r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161)
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
!importsOf(r.pkgs, repository) || only != nil && !only[module]) { !importsOf(r.pkgs, repository) || only != nil && !only[module]) {
// A module registered for another repository is moved by this repository's commit only when it imports a package the // A module registered for another repository is moved by this repository's commit only when it imports a package the
@@ -497,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool {
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
// module has none until they agree. // module has none until they agree.
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) { func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) {
if k.held.Fp2 == "" { if k.held.Fp2 == "" {
return "", nil, false return "", nil, false
} }
@@ -514,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
} }
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil)
return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2 if f.none != "" || f.fp == "" {
return "", nil, false
}
fp, pkgs := pinnedOf(f, held, released)
return fp, pkgs, fp == k.held.Fp2
} }
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
@@ -17,6 +17,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -783,15 +784,33 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
mods.put("mesh-tools", mustEncode(t, stale)) mods.put("mesh-tools", mustEncode(t, stale))
tools = stale tools = stale
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "")
ix.Versions = newMem() // mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) // pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A
// re-pin that fails is no release acknowledged: the package-published delivered again re-pins.
ix.Versions, ix.Heard = newMem(), newMem()
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`))
// mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165). revisions := ix.Revisions
if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { ix.Revisions = newMemObjects(1 << 20)
t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err) rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}})
if err != nil {
t.Fatal(err)
} }
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil {
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) t.Fatal(err)
}
release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"}
mods.keysErr = errors.New("the bus did not answer")
if err := ix.HearPackagePublished(context.Background(), release); err == nil {
t.Fatal("a release whose re-pin failed was acknowledged")
}
mods.keysErr = nil
if err := ix.HearPackagePublished(context.Background(), release); err != nil {
t.Fatal(err)
}
ix.Revisions = revisions
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) {
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages)
} }
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 {
t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2)
@@ -127,8 +127,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
return err return err
} else if found && !isFloor(raw) { } else if found && !isFloor(raw) {
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) // A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else.
return nil ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact)
return ix.repin(ctx, p.Module, p.Version)
} }
entry, err := ix.entryOf(ctx, p.Module) entry, err := ix.entryOf(ctx, p.Module)
if err != nil { if err != nil {
@@ -238,11 +239,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
} }
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
} }
if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165) if !late {
if n, err := ix.Repin(ctx, p.Module); err != nil { // The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not
ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err) // acknowledged, so the package-published is delivered again (H-A-167).
} else if n > 0 { if err := ix.repin(ctx, p.Module, p.Version); err != nil {
ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version) return err
} }
} }
if err := ix.trimVersions(ctx, p.Module); err != nil { if err := ix.trimVersions(ctx, p.Module); err != nil {
@@ -399,11 +400,14 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.
} }
out := sdkstep.PackageVersions{} out := sdkstep.PackageVersions{}
for p, v := range versions { for p, v := range versions {
switch key := fingerprint.HolderOf(p, held); { rel := released[moduleOfKey(fingerprint.HolderOf(p, held))]
case sdkstep.PackageSemver.MatchString(v): series := sdkstep.PackageSemver.MatchString(v)
_, plain := versionParts(strings.TrimPrefix(v, "v"))
switch {
case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0):
out[p] = "v" + rel // never an older series' version over a newer release (H-A-167)
case series:
out[p] = v out[p] = v
case released[moduleOfKey(key)] != "":
out[p] = "v" + released[moduleOfKey(key)]
default: default:
out[p] = "" out[p] = ""
} }
@@ -411,6 +415,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.
return out return out
} }
// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the
// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is.
func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, held, released)
if !f.line {
return f.fp, pins
}
return fingerprint.WithPins(f.fp, pins), pins
}
// moduleOfKey is the module a modules key names; "" for none. // moduleOfKey is the module a modules key names; "" for none.
func moduleOfKey(key string) string { func moduleOfKey(key string) string {
if key == "" { if key == "" {
@@ -437,3 +452,15 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions {
} }
return out return out
} }
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
if n > 0 {
ix.logf("versions: %d modules re-pinned to %s %s", n, module, version)
}
return nil
}