From ced27928d93fd5c401fcac400a8aca3e744a028f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 02:54:58 +0100 Subject: [PATCH 1/2] Design 47 H1: the indexer publishes a repository's packages in the merge's tree (H-A-232, H-A-233, H-A-234) A Go module mesh-catalog removed stayed in the packages state, so every publish of mesh-catalog asked the forge for its directory, the seat git answered not-found, and the one-directory fallback (issue 1585) ran only on an answer without the directory: nothing of the repository was published and its packages' digests froze unsaid. filesVia takes the seat's not-found as a failure of the tree, so the fallback runs; publish drops the kept key of a package whose directory the trunk's merge no longer holds, said in the journal; a listing that fails for another reason raises the finding packages-not-published (scope merge, id the repository), cleared at the next publish that lists. Builds: design 47 H H-A-232, H-A-233, H-A-234; issue 1585 --- .../cmd/mesh-delivery-indexer/named.go | 8 +- .../cmd/mesh-delivery-indexer/named_test.go | 14 +++ .../internal/index/packages.go | 72 ++++++++++++-- .../internal/index/packages_gone_test.go | 96 +++++++++++++++++++ 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/packages_gone_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 04da28ad..98f5c206 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "path" "slices" @@ -115,10 +116,15 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) } // filesVia asks the forge for every blob under each directory at a commit (the seat git's trees with files, ADR 0593 §2), -// by directory, then path relative to it. An answer that lists no directory asked is an error naming it. +// by directory, then path relative to it. An answer that lists no directory asked is an error naming it. The seat answers +// a directory the commit does not hold with the kind not-found naming it (the contract's files field; design 47 H H-A-232): +// that is a failure of the tree too (index.ErrNotInTree), so the indexer reads each directory alone and leaves the gone one out. func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { a, err := treesOf(call, repository, git.TreesInput{Commit: sha, Files: dirs}) + if errors.Is(err, index.ErrNotFound) { + return nil, fmt.Errorf("%w: %w", err, index.ErrNotInTree) + } if err != nil { return nil, err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index cc42bf67..26ee4983 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -75,6 +75,20 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) { t.Errorf("a missing directory: %v, want ErrNotInTree", err) } + // H-A-232 (2): the seat's not-found naming a directory the commit lacks is a failure of the tree too, so the + // one-directory fallback runs; a failed listing of another kind is not. + gone := func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"error":"the directory modules/mesh-build-go/buildstep is not in novox/mesh-catalog at x","error-kind":"not-found"}`), nil + } + if _, err := filesVia(gone)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/mesh-build-go/buildstep"}); !errors.Is(err, index.ErrNotInTree) { + t.Errorf("a directory the seat says not-found: %v, want ErrNotInTree", err) + } + failed := func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"error":"the forge did not answer","error-kind":"failed"}`), nil + } + if _, err := filesVia(failed)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/a"}); err == nil || errors.Is(err, index.ErrNotInTree) { + t.Errorf("a listing that failed: %v, want an error that is not ErrNotInTree", err) + } down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") } if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) { t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err) diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index 1850df22..3b8b708b 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -14,6 +14,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" @@ -138,8 +140,14 @@ func reached(held map[string]meshcatalog.ModulesByModuleValue) (modules, imports // directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge // directory by directory). A module the listing has no go.mod for is said and left out. func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, error) { + out, _, err := ix.packagesIn(ctx, repository, commit, modules, imports) + return out, err +} + +// packagesIn is packagesAt, with the modules whose directory the tree at the commit does not hold (gone), by module path. +func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, []string, error) { if ix.Files == nil || commit == "" { - return nil, nil + return nil, nil, nil } dirOf := map[string]string{} // module path to its directory in the repository for _, m := range modules { @@ -182,7 +190,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo _, dir, _ := trunk.PackageRepository(imp) for d := dir; ; d = parent(d) { if err := list([]string{d}); err != nil { - return nil, err + return nil, nil, err } if _, ok := listings[d]["go.mod"]; ok { m := strings.TrimSuffix(trunk.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/") @@ -202,26 +210,28 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo dirs := slices.Sorted(maps.Values(dirOf)) if err := list(dirs); err != nil { if !errors.Is(err, ErrNotInTree) || len(dirs) < 2 { - return nil, err + return nil, nil, err } whole := err for _, d := range dirs { if err := list([]string{d}); err != nil { if !errors.Is(err, ErrNotInTree) { - return nil, err + return nil, nil, err } ix.logf("%s at %s: the directory %q is not in the tree, so the module there is not read: %v", repository, commit, d, err) absent[d] = true } } if len(absent) == len(dirs) { - return nil, whole // the tree itself fails + return nil, nil, whole // the tree itself fails } } out := map[string]meshcatalog.PackagesByModuleValue{} + var gone []string for _, m := range slices.Sorted(maps.Keys(dirOf)) { dir := dirOf[m] if absent[dir] { + gone = append(gone, m) continue } requires, pkgs, err := fingerprint.Packages(m, listings[dir]) @@ -236,7 +246,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo out[m] = meshcatalog.PackagesByModuleValue{Module: m, Repository: trunk.RepositoryName(repository), Dir: d, Commit: commit, Requires: requires, Packages: pkgs} } - return out, nil + return out, gone, nil } // parent is the directory above d, "" for the repository's root. @@ -287,10 +297,28 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held modules[repo] = append(modules[repo], m) } } - got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo]) + got, gone, err := ix.packagesIn(ctx, repo, commit, modules[repo], imports[repo]) if err != nil { + ix.notPublished(ctx, repo, commit, err) return err } + ix.published(ctx, repo, commit) + // A repository's packages are those in the merge's tree (design 47 H H-A-232): the kept key of a package whose directory + // the trunk's merge no longer holds is dropped, said, so the forge is never asked for it again. + for _, m := range gone { + if _, kept := have[m]; !kept { + continue + } + key, err := PackagesKey(m) + if err != nil { + ix.logf("%s: %v", m, err) + continue + } + if err := write(ctx, ix.Packages, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + return fmt.Errorf("packages.%s: %w", key, err) + } + ix.logf("package %s of %s: its directory is gone at %s: its kept key dropped", m, repo, commit) + } for _, m := range slices.Sorted(maps.Keys(got)) { next := got[m] key, err := PackagesKey(m) @@ -316,6 +344,36 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held return nil } +// PackagesNotPublishedKind is the finding of a repository whose packages a take-in's publish could not list, for a reason +// that is not a gone directory (design 47 H H-A-233, H-A-234): scope merge, id the repository, with the forge's words; +// raised at once and cleared at the repository's next publish that lists. +const PackagesNotPublishedKind = "packages-not-published" + +// notPublished raises the finding packages-not-published of repository. +func (ix *Indexer) notPublished(ctx context.Context, repository, commit string, cause error) { + if ix.Findings == nil { + return + } + f := ix.observed(mcc.ScopeMerge, repository, PackagesNotPublishedKind, + fmt.Sprintf("the packages of %s are not published at %.12s", repository, commit), + "A repository's packages are not published", + fmt.Sprintf("The take-in of %s at %s could not list its packages (%v), so their digests stay at their last published commit and a change to one does not move its importers.", repository, commit, cause), + "indexer."+PackagesNotPublishedKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("%s: the finding %s is not written: %v", repository, PackagesNotPublishedKind, err) + } +} + +// published clears the finding packages-not-published of repository: its publish listed. +func (ix *Indexer) published(ctx context.Context, repository, commit string) { + if ix.Findings == nil { + return + } + if err := ix.clearFinding(ctx, mcc.ScopeMerge, repository, PackagesNotPublishedKind, "its packages are listed at "+commit); err != nil { + ix.logf("%s: the finding %s is not cleared: %v", repository, PackagesNotPublishedKind, err) + } +} + // FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the // packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of. // A repository with no taken merge is left for its next merge. diff --git a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go new file mode 100644 index 00000000..43321cdb --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go @@ -0,0 +1,96 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// Design 47 H H-A-232 (1), H-A-233, H-A-234: a repository's packages are those in the merge's tree. The kept key of a +// package whose directory the trunk's merge no longer holds is dropped, said, and the repository's other packages are +// published all the same (issue 1585's one-directory fallback); a listing that fails for another reason raises the +// finding packages-not-published (scope merge, id the repository), cleared at the next publish that lists. +func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { + const ( + live = "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings" + gone = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + liveDir = "modules/disk-load/diskreadings" + ) + ctx := context.Background() + old, merge := strings.Repeat("d", 40), strings.Repeat("e", 40) + held := map[string]meshcatalog.ModulesByModuleValue{ + "disk-load": {Repository: catalogueRepo, Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{live}}}, + } + var down error // the forge's listing fails for a reason that is not a gone directory + files := func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { + if down != nil { + return nil, down + } + out := map[string]map[string]string{} + for _, d := range dirs { + if d != liveDir { + // as filesVia says the seat's not-found naming the directory + return nil, fmt.Errorf("the directory %s is not in %s at %s: %w", d, repository, sha, ErrNotInTree) + } + out[d] = map[string]string{"go.mod": "m1", "readings.go": "r-" + sha[:1]} + } + return out, nil + } + var said []string + findings := newFindings() + ix := &Indexer{Module: testSelf, Packages: newMem(), Files: files, Findings: findings, + Logf: func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }} + for _, m := range []struct{ module, dir string }{{live, liveDir}, {gone, "modules/mesh-build-go/buildstep"}} { + key, err := PackagesKey(m.module) + if err != nil { + t.Fatal(err) + } + raw, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: m.module, Repository: catalogueRepo, Dir: m.dir, Commit: old, Requires: "req", At: ptr(time.Unix(0, 0).UTC()), + Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: m.module, Digest: "x"}}}) + if err != nil { + t.Fatal(err) + } + ix.Packages.(*memStore).put(key, raw) + } + notPublished := func() bool { + for _, f := range findings.Held() { + if f.Kind == PackagesNotPublishedKind && f.Scope == mcc.ScopeMerge && f.ID == keyID(catalogueRepo) { + return true + } + } + return false + } + + down = errors.New("seat:git.trees (failed): the forge did not answer") + if err := ix.publish(ctx, catalogueRepo, merge, held, false); err == nil { + t.Fatal("a listing that failed published") + } + if !notPublished() { + t.Fatalf("a listing that failed raised no finding %s: %v", PackagesNotPublishedKind, findings.Held()) + } + + down = nil + if err := ix.publish(ctx, catalogueRepo, merge, held, false); err != nil { + t.Fatal(err) + } + have := ix.publishedAll(ctx) + if have[live].Commit != merge { + t.Errorf("packages of %s at %q, want the merge %s: a gone sibling kept it from being published", live, have[live].Commit, merge) + } + if _, kept := have[gone]; kept { + t.Errorf("the kept key of %s, whose directory the merge does not hold, stays", gone) + } + want := "package " + gone + " of " + catalogueRepo + ": its directory is gone at " + merge + ": its kept key dropped" + if !strings.Contains(strings.Join(said, "\n"), want) { + t.Errorf("the journal does not say %q:\n%s", want, strings.Join(said, "\n")) + } + if notPublished() { + t.Error("the finding packages-not-published stays after a publish that listed") + } +} From 588a6ff8d52fd2a5bdba749a8718170eeecedb14 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 03:07:02 +0100 Subject: [PATCH 2/2] Design 47 H1: only the trunk's merge drops a gone package; a commit's tree tells an all-gone repository (H-A-232, H-A-233) Review round 1. A branch's take-in runs through the dry indexer, which shares the live packages state, so a branch cut before main added a module would have deleted main's key: publish drops a kept key only on the real indexer at the trunk's merge. And a repository whose every asked directory is gone (one key, or all gone) returned the whole error and raised packages-not-published at every take-in: the seat's trees of the commit now tell a gone directory from a commit with no tree. Builds: design 47 H H-A-232, H-A-233, H-A-234; issue 1585 --- .../internal/index/candidates.go | 1 + .../internal/index/dryrun.go | 2 +- .../internal/index/imports_test.go | 14 +- .../internal/index/modules.go | 2 +- .../internal/index/packages.go | 28 ++- .../internal/index/packages_gone_test.go | 186 ++++++++++++------ 6 files changed, 159 insertions(+), 74 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 6ca63407..b880cdb9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -156,6 +156,7 @@ type Indexer struct { filling atomic.Bool // a FillTaken runs in the background (KickFill) fillAgain atomic.Bool // a kick arrived while it ran underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + isDry bool // a dry run (dryrun.go): it shares the live packages state, so it drops no kept package key (packages.go) judgeMu sync.Mutex // one judge pass at a time (verdicts.go) keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) // Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go index ab1ee7d3..bdf7a4bd 100644 --- a/modules/mesh-delivery-indexer/internal/index/dryrun.go +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -35,7 +35,7 @@ func (ix *Indexer) dry() *dryRun { // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - Heard: nil, underSeats: true, + Heard: nil, underSeats: true, isDry: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, InputObjects: nil, Pins: nil, PinsPending: nil, // A dry run writes no registration and asks no one. diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 7bca9361..65876c13 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -68,7 +68,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T } mods.put("mesh-sdk", sdkModule) held, _ := ix.heldModules(ctx) - if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil { + if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false, TrunkBranch); err != nil { t.Fatal(err) } pub := ix.publishedAll(ctx)[meshSDK] @@ -99,7 +99,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T // mesh-sdk merges a change of buildstep only: avahi does not import it. held, _ = ix.heldModules(ctx) - if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false); err != nil { + if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false, TrunkBranch); err != nil { t.Fatal(err) } if err := ix.Recompute(ctx); err != nil { @@ -111,7 +111,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T // mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version. held, _ = ix.heldModules(ctx) - if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil { + if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false, TrunkBranch); err != nil { t.Fatal(err) } if err := ix.Recompute(ctx); err != nil { @@ -195,7 +195,7 @@ func aSiblingDirectoryAloneMovesItsImporter(t *testing.T) { fpAt := func(commit string) meshcatalog.ModulesByModuleValue { t.Helper() held, _ := ix.heldModules(ctx) - if err := ix.publish(ctx, catalogueRepo, commit, held, false); err != nil { + if err := ix.publish(ctx, catalogueRepo, commit, held, false, TrunkBranch); err != nil { t.Fatal(err) } if err := ix.Recompute(ctx); err != nil { @@ -274,7 +274,7 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) { trunk := func(commit string) func() error { return func() error { held, _ := ix.heldModules(ctx) - return ix.publish(ctx, "novox/mesh-sdk", commit, held, false) + return ix.publish(ctx, "novox/mesh-sdk", commit, held, false, TrunkBranch) } } hear := func(version, commit string) func() error { @@ -328,7 +328,7 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * } mods.put("mesh-sdk", raw) held, _ := ix.heldModules(ctx) - if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil { + if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false, TrunkBranch); err != nil { t.Fatal(err) } if err := ix.Recompute(ctx); err != nil { @@ -339,7 +339,7 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * t.Fatal(err) } held, _ = ix.heldModules(ctx) - if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil { + if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false, TrunkBranch); err != nil { t.Fatal(err) } if err := ix.Recompute(ctx); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index a67cd6b7..9a59b83d 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -156,7 +156,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma // 0593 §3). A forge that cannot list them leaves the state as it was, said: the importers move at the next merge. reach := maps.Clone(held) maps.Copy(reach, overlay) - if err := ix.publish(ctx, f.Repository, f.Merge, reach, false); err != nil { + if err := ix.publish(ctx, f.Repository, f.Merge, reach, false, f.Branch); err != nil { ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err) } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index 3b8b708b..d898e2c9 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -209,12 +209,15 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo absent := map[string]bool{} dirs := slices.Sorted(maps.Values(dirOf)) if err := list(dirs); err != nil { - if !errors.Is(err, ErrNotInTree) || len(dirs) < 2 { + if !errors.Is(err, ErrNotInTree) { return nil, nil, err } whole := err for _, d := range dirs { - if err := list([]string{d}); err != nil { + if len(dirs) > 1 { + err = list([]string{d}) + } + if err != nil { if !errors.Is(err, ErrNotInTree) { return nil, nil, err } @@ -222,8 +225,16 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo absent[d] = true } } + // Every directory asked is missing: gone only when the commit holds a tree at all (the seat's trees of the commit, no + // files), else the tree itself fails (H-A-232, H-A-233). if len(absent) == len(dirs) { - return nil, nil, whole // the tree itself fails + if ix.Trunk == nil { + return nil, nil, whole + } + if _, err := ix.Trunk(ctx, repository, commit); err != nil { + return nil, nil, fmt.Errorf("%w (and the commit's trees: %v)", whole, err) + } + ix.logf("%s at %s: the commit holds a tree, so every directory asked (%s) is gone", repository, commit, strings.Join(dirs, ", ")) } } out := map[string]meshcatalog.PackagesByModuleValue{} @@ -262,8 +273,10 @@ func parent(d string) string { } // publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach, -// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left. -func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool) error { +// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left. The kept +// key of a package whose directory the commit does not hold is dropped at the trunk's merge on the real indexer only, never +// at a branch's take-in (branch) or in a dry run, which share the live packages state (H-A-232 (1)). +func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool, branch string) error { if ix.Packages == nil || ix.Files == nil { return nil } @@ -273,6 +286,7 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held return nil } have := ix.publishedAll(ctx) + drop := !ix.isDry && isTrunkBranch(branch) if onlyMissing { // Imports a held module already resolves are not asked again. var ms, is []string @@ -306,7 +320,7 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held // A repository's packages are those in the merge's tree (design 47 H H-A-232): the kept key of a package whose directory // the trunk's merge no longer holds is dropped, said, so the forge is never asked for it again. for _, m := range gone { - if _, kept := have[m]; !kept { + if _, kept := have[m]; !kept || !drop { continue } key, err := PackagesKey(m) @@ -407,7 +421,7 @@ func (ix *Indexer) FillPackages(ctx context.Context) error { continue } began := time.Now() - if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil { + if err := ix.publish(ctx, repo, t.Merge, held, true, TrunkBranch); err != nil { ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err) } ix.logf("publishing the packages at start: %s at %.12s took %s", repo, t.Merge, time.Since(began).Round(time.Millisecond)) diff --git a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go index 43321cdb..56ac91e8 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go +++ b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go @@ -14,83 +14,153 @@ import ( // Design 47 H H-A-232 (1), H-A-233, H-A-234: a repository's packages are those in the merge's tree. The kept key of a // package whose directory the trunk's merge no longer holds is dropped, said, and the repository's other packages are -// published all the same (issue 1585's one-directory fallback); a listing that fails for another reason raises the -// finding packages-not-published (scope merge, id the repository), cleared at the next publish that lists. +// published all the same (issue 1585's one-directory fallback); a branch's take-in and a dry run, which share the live +// packages state, drop nothing. A repository whose every asked directory is gone, at a commit that holds a tree, drops them +// and raises nothing; a listing that fails for another reason, or a commit with no tree, raises the finding +// packages-not-published (scope merge, id the repository), cleared at the next publish that lists. func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { const ( live = "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings" gone = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" liveDir = "modules/disk-load/diskreadings" + goneDir = "modules/mesh-build-go/buildstep" ) ctx := context.Background() old, merge := strings.Repeat("d", 40), strings.Repeat("e", 40) - held := map[string]meshcatalog.ModulesByModuleValue{ - "disk-load": {Repository: catalogueRepo, Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{live}}}, - } - var down error // the forge's listing fails for a reason that is not a gone directory - files := func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { - if down != nil { - return nil, down + reaching := func(m string) map[string]meshcatalog.ModulesByModuleValue { + return map[string]meshcatalog.ModulesByModuleValue{ + "user": {Repository: catalogueRepo, Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{m}}}, } - out := map[string]map[string]string{} - for _, d := range dirs { - if d != liveDir { - // as filesVia says the seat's not-found naming the directory - return nil, fmt.Errorf("the directory %s is not in %s at %s: %w", d, repository, sha, ErrNotInTree) + } + type setup struct { + ix *Indexer + findings *memFindings + said *[]string + down *error // the forge's listing fails for a reason that is not a gone directory + noTree *error // the seat's trees of the commit, without files, fail + } + newSetup := func(t *testing.T, kept ...string) setup { + t.Helper() + var down, noTree error + said := []string{} + files := func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { + if down != nil { + return nil, down } - out[d] = map[string]string{"go.mod": "m1", "readings.go": "r-" + sha[:1]} + out := map[string]map[string]string{} + for _, d := range dirs { + if d != liveDir { + // as filesVia says the seat's not-found naming the directory + return nil, fmt.Errorf("the directory %s is not in %s at %s: %w", d, repository, sha, ErrNotInTree) + } + out[d] = map[string]string{"go.mod": "m1", "readings.go": "r-" + sha[:1]} + } + return out, nil } - return out, nil + trees := func(context.Context, string, string) ([]string, error) { return []string{"modules/disk-load"}, noTree } + findings := newFindings() + ix := &Indexer{Module: testSelf, Packages: newMem(), Files: files, Trunk: trees, Findings: findings, + Logf: func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }} + dirOf := map[string]string{live: liveDir, gone: goneDir} + for _, m := range kept { + key, err := PackagesKey(m) + if err != nil { + t.Fatal(err) + } + raw, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: m, Repository: catalogueRepo, Dir: dirOf[m], Commit: old, + Requires: "req", At: ptr(time.Unix(0, 0).UTC()), Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: m, Digest: "x"}}}) + if err != nil { + t.Fatal(err) + } + ix.Packages.(*memStore).put(key, raw) + } + return setup{ix: ix, findings: findings, said: &said, down: &down, noTree: &noTree} } - var said []string - findings := newFindings() - ix := &Indexer{Module: testSelf, Packages: newMem(), Files: files, Findings: findings, - Logf: func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }} - for _, m := range []struct{ module, dir string }{{live, liveDir}, {gone, "modules/mesh-build-go/buildstep"}} { - key, err := PackagesKey(m.module) - if err != nil { - t.Fatal(err) - } - raw, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: m.module, Repository: catalogueRepo, Dir: m.dir, Commit: old, Requires: "req", At: ptr(time.Unix(0, 0).UTC()), - Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: m.module, Digest: "x"}}}) - if err != nil { - t.Fatal(err) - } - ix.Packages.(*memStore).put(key, raw) - } - notPublished := func() bool { - for _, f := range findings.Held() { + notPublished := func(s setup) bool { + for _, f := range s.findings.Held() { if f.Kind == PackagesNotPublishedKind && f.Scope == mcc.ScopeMerge && f.ID == keyID(catalogueRepo) { return true } } return false } + dropped := "package " + gone + " of " + catalogueRepo + ": its directory is gone at " + merge + ": its kept key dropped" - down = errors.New("seat:git.trees (failed): the forge did not answer") - if err := ix.publish(ctx, catalogueRepo, merge, held, false); err == nil { - t.Fatal("a listing that failed published") - } - if !notPublished() { - t.Fatalf("a listing that failed raised no finding %s: %v", PackagesNotPublishedKind, findings.Held()) + t.Run("the trunk's merge drops a gone package's key and publishes the others; a failed listing is a finding until one lists", func(t *testing.T) { + s := newSetup(t, live, gone) + *s.down = errors.New("seat:git.trees (failed): the forge did not answer") + if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(live), false, TrunkBranch); err == nil { + t.Fatal("a listing that failed published") + } + if !notPublished(s) { + t.Fatalf("a listing that failed raised no finding %s: %v", PackagesNotPublishedKind, s.findings.Held()) + } + *s.down = nil + if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(live), false, TrunkBranch); err != nil { + t.Fatal(err) + } + have := s.ix.publishedAll(ctx) + if have[live].Commit != merge { + t.Errorf("packages of %s at %q, want the merge %s: a gone sibling kept it from being published", live, have[live].Commit, merge) + } + if _, kept := have[gone]; kept { + t.Errorf("the kept key of %s, whose directory the merge does not hold, stays", gone) + } + if !strings.Contains(strings.Join(*s.said, "\n"), dropped) { + t.Errorf("the journal does not say %q:\n%s", dropped, strings.Join(*s.said, "\n")) + } + if notPublished(s) { + t.Error("the finding packages-not-published stays after a publish that listed") + } + }) + + for _, row := range []struct { + name string + dry bool + branch string + }{ + {"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, "feat/x"}, + {"a dry run keeps the key: it shares the live packages state", true, TrunkBranch}, + } { + t.Run(row.name, func(t *testing.T) { + s := newSetup(t, live, gone) + ix := s.ix + if row.dry { + ix = s.ix.dry().ix + } + if err := ix.publish(ctx, catalogueRepo, merge, reaching(live), false, row.branch); err != nil { + t.Fatal(err) + } + if _, kept := s.ix.publishedAll(ctx)[gone]; !kept { + t.Errorf("the kept key of %s was dropped", gone) + } + }) } - down = nil - if err := ix.publish(ctx, catalogueRepo, merge, held, false); err != nil { - t.Fatal(err) - } - have := ix.publishedAll(ctx) - if have[live].Commit != merge { - t.Errorf("packages of %s at %q, want the merge %s: a gone sibling kept it from being published", live, have[live].Commit, merge) - } - if _, kept := have[gone]; kept { - t.Errorf("the kept key of %s, whose directory the merge does not hold, stays", gone) - } - want := "package " + gone + " of " + catalogueRepo + ": its directory is gone at " + merge + ": its kept key dropped" - if !strings.Contains(strings.Join(said, "\n"), want) { - t.Errorf("the journal does not say %q:\n%s", want, strings.Join(said, "\n")) - } - if notPublished() { - t.Error("the finding packages-not-published stays after a publish that listed") - } + t.Run("a one-key repository whose directory is gone drops it and raises nothing, when the commit holds a tree", func(t *testing.T) { + s := newSetup(t, gone) + if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(gone), false, TrunkBranch); err != nil { + t.Fatal(err) + } + if _, kept := s.ix.publishedAll(ctx)[gone]; kept { + t.Errorf("the kept key of %s, the repository's one package, whose directory is gone, stays", gone) + } + if notPublished(s) { + t.Error("a gone directory raised packages-not-published") + } + }) + + t.Run("every asked directory missing at a commit with no tree is the tree's failure: the keys stay and it is a finding", func(t *testing.T) { + s := newSetup(t, gone) + *s.noTree = errors.New("seat:git.trees (not-found): Gitea API GET: 404 Not Found") + if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(gone), false, TrunkBranch); err == nil { + t.Fatal("a commit with no tree published") + } + if _, kept := s.ix.publishedAll(ctx)[gone]; !kept { + t.Errorf("the kept key of %s was dropped at a commit with no tree", gone) + } + if !notPublished(s) { + t.Error("a commit with no tree raised no finding") + } + }) }