diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index c5a4e5cb..64bf2d12 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -481,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) { // NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own // git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree). const NamedTreePrefix = "named:" + +// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no +// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin " of each pin, +// joined by a newline. A package kept with no version (none released) is no pin. +func WithPins(fp string, pins sdkstep.PackageVersions) string { + var lines []string + for p, v := range pins { + if v != "" { + lines = append(lines, "pin "+p+" "+v) + } + } + if fp == "" || len(lines) == 0 { + return fp + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n"))) + return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:]) +} diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index e638c374..28b92d33 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -17,6 +17,7 @@ import ( type form struct { fp, none string versions sdkstep.PackageVersions + line bool // the version line's, whose fingerprint takes the pins too (pinnedOf, H-A-167) } // formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the @@ -71,7 +72,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp if fp == "" { return form{none: none} } - return form{fp: fp, versions: versions} + return form{fp: fp, versions: versions, line: true} } // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 222cf39f..6371753a 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -272,6 +272,12 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" { t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages) } + // mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167). + ix.Versions = newMem() + ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" { + t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages) + } } // Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 22c7b3a9..6824bb67 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, - Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161) Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) if err != nil { @@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the image artifact is %+v", a) } in := got[0] - if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" { t.Errorf("the steps input is %+v", in) } + if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned { + t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages) + } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 94bec3d7..f3b72e2f 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -291,17 +291,18 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { - r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions + // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) + r.fp, r.pkgs = pinnedOf(f, held, released) + r.tcs = toolchainsUsed(toolchains) } } - r.pkgs = releasedPins(r.pkgs, held, released) // a pin is a released version, never a commit (H-A-161) if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) && !importsOf(r.pkgs, repository) || only != nil && !only[module]) { // A module registered for another repository is moved by this repository's commit only when it imports a package the @@ -497,7 +498,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) { +func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) { if k.held.Fp2 == "" { return "", nil, false } @@ -514,7 +515,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) - return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2 + if f.none != "" || f.fp == "" { + return "", nil, false + } + fp, pkgs := pinnedOf(f, held, released) + return fp, pkgs, fp == k.held.Fp2 } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index b35cb8ec..281540a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -783,15 +784,33 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { mods.put("mesh-tools", mustEncode(t, stale)) tools = stale first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "") - ix.Versions = newMem() - ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`)) + // mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the + // pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A + // re-pin that fails is no release acknowledged: the package-published delivered again re-pins. + ix.Versions, ix.Heard = newMem(), newMem() ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`)) - // mesh-sdk's release re-pins avahi with no merge, and leaves mesh-tools, which holds none of it (H-A-165). - if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { - t.Fatalf("re-pin on mesh-sdk's release: %d modules, %v; want avahi alone", n, err) + revisions := ix.Revisions + ix.Revisions = newMemObjects(1 << 20) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}}) + if err != nil { + t.Fatal(err) } - if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 != first.Fp2 || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { - t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want the same fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) + if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"} + mods.keysErr = errors.New("the bus did not answer") + if err := ix.HearPackagePublished(context.Background(), release); err == nil { + t.Fatal("a release whose re-pin failed was acknowledged") + } + mods.keysErr = nil + if err := ix.HearPackagePublished(context.Background(), release); err != nil { + t.Fatal(err) + } + ix.Revisions = revisions + if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) { + t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages) } if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 { t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2) diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index d82709e4..0587b99d 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -127,8 +127,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err } else if found && !isFloor(raw) { - ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) - return nil + // A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else. + ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact) + return ix.repin(ctx, p.Module, p.Version) } entry, err := ix.entryOf(ctx, p.Module) if err != nil { @@ -238,11 +239,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli } ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source) } - if !late { // the release re-pins every module whose packages hold one of it (design 47 H-A-165) - if n, err := ix.Repin(ctx, p.Module); err != nil { - ix.logf("versions: the modules pinned to %s are not re-pinned to %s: %v", p.Module, p.Version, err) - } else if n > 0 { - ix.logf("versions: %d modules re-pinned to %s %s", n, p.Module, p.Version) + if !late { + // The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not + // acknowledged, so the package-published is delivered again (H-A-167). + if err := ix.repin(ctx, p.Module, p.Version); err != nil { + return err } } if err := ix.trimVersions(ctx, p.Module); err != nil { @@ -399,11 +400,14 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. } out := sdkstep.PackageVersions{} for p, v := range versions { - switch key := fingerprint.HolderOf(p, held); { - case sdkstep.PackageSemver.MatchString(v): + rel := released[moduleOfKey(fingerprint.HolderOf(p, held))] + series := sdkstep.PackageSemver.MatchString(v) + _, plain := versionParts(strings.TrimPrefix(v, "v")) + switch { + case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0): + out[p] = "v" + rel // never an older series' version over a newer release (H-A-167) + case series: out[p] = v - case released[moduleOfKey(key)] != "": - out[p] = "v" + released[moduleOfKey(key)] default: out[p] = "" } @@ -411,6 +415,17 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog. return out } +// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so the +// version line's fingerprint takes them (design 47 H-A-167, ADR 0573 ยง4) and a re-pin moves it, the module a mover +// built again against the release; a fingerprint with no pin is the form's as it is. +func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, held, released) + if !f.line { + return f.fp, pins + } + return fingerprint.WithPins(f.fp, pins), pins +} + // moduleOfKey is the module a modules key names; "" for none. func moduleOfKey(key string) string { if key == "" { @@ -437,3 +452,15 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { } return out } + +// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. +func (ix *Indexer) repin(ctx context.Context, module, version string) error { + n, err := ix.Repin(ctx, module) + if err != nil { + return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) + } + if n > 0 { + ix.logf("versions: %d modules re-pinned to %s %s", n, module, version) + } + return nil +}