From fe51379ee2514b3d03f98636a9d89c1ec09615e1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:44:13 +0100 Subject: [PATCH] mesh-delivery-indexer: a release re-pins or is not acknowledged; pins enter the version line's fingerprint A re-pin that left the fingerprint as it was left the build made against the older pin as the one the mesh runs. The version line's fingerprint now takes the pins, so a re-pin moves the importer, which becomes a mover built against the release. An accepted series' older version no longer stands over a newer release. A re-pin that fails returns its error before the heard artifacts are forgotten, and a package-published heard again for a released version re-pins, so the redelivery recovers it. The release path and the seed's pins carry tests that fail without them (review round 2). Builds: design 47 H H-A-165, H-A-167, H-A-161 --- .../internal/fingerprint/fingerprint.go | 18 +++++++ .../internal/index/form.go | 3 +- .../internal/index/imports_test.go | 6 +++ .../internal/index/seed_steps_test.go | 7 ++- .../internal/index/toolchains.go | 15 ++++-- .../internal/index/toolchains_test.go | 33 ++++++++++--- .../internal/index/versions.go | 49 ++++++++++++++----- 7 files changed, 105 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index c5a4e5cb..64bf2d12 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -481,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { // NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own // git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree). const NamedTreePrefix = "named:" + +// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no +// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin " of each pin, +// joined by a newline. A package kept with no version (none released) is no pin. +func WithPins(fp string, pins sdkstep.PackageVersions) string { + var lines []string + for p, v := range pins { + if v != "" { + lines = append(lines, "pin "+p+" "+v) + } + } + if fp == "" || len(lines) == 0 { + return fp + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n"))) + return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:]) +} diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index e638c374..28b92d33 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -17,6 +17,7 @@ import ( type form struct { fp, none string versions sdkstep.PackageVersions + line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167) } // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the @@ -71,7 +72,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if fp == "" { return form{none: none} } - return form{fp: fp, versions: versions} + return form{fp: fp, versions: versions, line: true} } // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 222cf39f..6371753a 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -272,6 +272,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" { t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages) } + // mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" { + t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages) + } } // Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 22c7b3a9..6824bb67 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, - Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161) Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) if err != nil { @@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the image artifact is %+v", a) } in := got[0] - if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" { t.Errorf("the steps input is %+v", in) } + if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned { + t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages) + } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 94bec3d7..f3b72e2f 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -291,17 +291,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { - r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions + // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) + r.fp, r.pkgs = pinnedOf(f, held, released) + r.tcs = toolchainsUsed(toolchains) } } - r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && !importsOf(r.pkgs, repository) || only != nil && !only[module]) { // A module registered for another repository is moved by this repository's commit only when it imports a package the @@ -497,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) { +func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) { if k.held.Fp2 == "" { return "", nil, false } @@ -514,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) - return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2 + if f.none != "" || f.fp == "" { + return "", nil, false + } + fp, pkgs := pinnedOf(f, held, released) + return fp, pkgs, fp == k.held.Fp2 } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index b35cb8ec..281540a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -783,15 +784,33 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { mods.put("mesh-tools", mustEncode(t, stale)) tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") - ix.Versions = newMem() - ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + // mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the + // pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A + // re-pin that fails is no release acknowledged: the package-published delivered again re-pins. + ix.Versions, ix.Heard = newMem(), newMem() ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) - // mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165). - if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { - t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err) + revisions := ix.Revisions + ix.Revisions = newMemObjects(1 << 20) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}}) + if err != nil { + t.Fatal(err) } - if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { - t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) + if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"} + mods.keysErr = errors.New("the bus did not answer") + if err := ix.HearPackagePublished(context.Background(), release); err == nil { + t.Fatal("a release whose re-pin failed was acknowledged") + } + mods.keysErr = nil + if err := ix.HearPackagePublished(context.Background(), release); err != nil { + t.Fatal(err) + } + ix.Revisions = revisions + if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index d82709e4..0587b99d 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -127,8 +127,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err } else if found && !isFloor(raw) { - ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) - return nil + // A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else. + ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact) + return ix.repin(ctx, p.Module, p.Version) } entry, err := ix.entryOf(ctx, p.Module) if err != nil { @@ -238,11 +239,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli } ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) } - if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165) - if n, err := ix.Repin(ctx, p.Module); err != nil { - ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err) - } else if n > 0 { - ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version) + if !late { + // The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not + // acknowledged, so the package-published is delivered again (H-A-167). + if err := ix.repin(ctx, p.Module, p.Version); err != nil { + return err } } if err := ix.trimVersions(ctx, p.Module); err != nil { @@ -399,11 +400,14 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. } out := sdkstep.PackageVersions{} for p, v := range versions { - switch key := fingerprint.HolderOf(p, held); { - case sdkstep.PackageSemver.MatchString(v): + rel := released[moduleOfKey(fingerprint.HolderOf(p, held))] + series := sdkstep.PackageSemver.MatchString(v) + _, plain := versionParts(strings.TrimPrefix(v, "v")) + switch { + case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0): + out[p] = "v" + rel // never an older series' version over a newer release (H-A-167) + case series: out[p] = v - case released[moduleOfKey(key)] != "": - out[p] = "v" + released[moduleOfKey(key)] default: out[p] = "" } @@ -411,6 +415,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. return out } +// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the +// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 ยง4) and a re-pin moves it, the module a mover +// built again against the release; a fingerprint with no pin is the form's as it is. +func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, held, released) + if !f.line { + return f.fp, pins + } + return fingerprint.WithPins(f.fp, pins), pins +} + // moduleOfKey is the module a modules key names; "" for none. func moduleOfKey(key string) string { if key == "" { @@ -437,3 +452,15 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { } return out } + +// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. +func (ix *Indexer) repin(ctx context.Context, module, version string) error { + n, err := ix.Repin(ctx, module) + if err != nil { + return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) + } + if n > 0 { + ix.logf("versions: %d modules re-pinned to %s %s", n, module, version) + } + return nil +}