Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/packages.go
T
jschoubben ced27928d9 Design 47 H1: the indexer publishes a repository's packages in the merge's tree (H-A-232, H-A-233, H-A-234)
A Go module mesh-catalog removed stayed in the packages state, so every
publish of mesh-catalog asked the forge for its directory, the seat git
answered not-found, and the one-directory fallback (issue 1585) ran only
on an answer without the directory: nothing of the repository was
published and its packages' digests froze unsaid.

filesVia takes the seat's not-found as a failure of the tree, so the
fallback runs; publish drops the kept key of a package whose directory
the trunk's merge no longer holds, said in the journal; a listing that
fails for another reason raises the finding packages-not-published
(scope merge, id the repository), cleared at the next publish that lists.

Builds: design 47 H H-A-232, H-A-233, H-A-234; issue 1585
2026-11-21 02:54:58 +01:00

421 lines
15 KiB
Go

package index
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/process"
"maps"
"path"
"reflect"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"mesh-delivery-indexer/internal/fingerprint"
)
// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
// digests in its state `packages`, one key per Go module. The builder, the checker and the
// build modules read it there; nobody else computes a digest or a version.
//
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
// reported (KeptInputs2.Imports). Its key is written at each merge of its repository, before the merge's recompute, so
// a merge that changes a package moves its importers in the same recompute (§3); and once at the start, for each
// reached module the state does not hold yet (FillPackages).
// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
func (ix *Indexer) publishedAll(ctx context.Context) map[string]meshcatalog.PackagesByModuleValue {
out, err := ix.publishedRead(ctx)
if err != nil {
ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err)
}
return out
}
// publishedRead is publishedAll with a read that failed said as an error, never as an empty map: a pass that takes an
// unread state for "no package is published" writes the version line over every module's fingerprint (issue 1966, the
// indexer started while the bus's storage was down). No state wired is no error: nothing is published then.
func (ix *Indexer) publishedRead(ctx context.Context) (map[string]meshcatalog.PackagesByModuleValue, error) {
out := map[string]meshcatalog.PackagesByModuleValue{}
if ix.Packages == nil {
return out, nil
}
keys, err := ix.Packages.Keys(ctx)
if errors.Is(err, process.ErrNotYet) {
return out, nil // the state is not on the bus yet: nothing is published, as before
}
if err != nil {
return out, fmt.Errorf("the packages state cannot be listed: %w", err)
}
for _, key := range keys {
if IsSeriesKey(key) {
continue // a package's series (series.go), not a Go module's digests
}
value, _, found, err := ix.Packages.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
continue
}
if err != nil {
return out, fmt.Errorf("packages.%s cannot be read: %w", key, err)
}
if !found {
continue
}
p, err := DecodePublishedPackages(value)
if err != nil {
ix.logf("packages.%s holds what is not a published module, left out: %v", key, err)
continue
}
out[p.Module] = p
}
return out, nil
}
// publishedForPass is the packages state a recompute computes fingerprints from, or why it must write none: a read that
// failed, or that answered no published module where the held modules show packages were published before (a module
// whose fingerprint was computed from the import form, or that holds resolved package versions). The first start of a
// mesh, whose held modules show none, computes from an empty state as before.
func (ix *Indexer) publishedForPass(ctx context.Context, held map[string]meshcatalog.ModulesByModuleValue) (map[string]meshcatalog.PackagesByModuleValue, error) {
published, err := ix.publishedRead(ctx)
if err != nil {
return nil, err
}
if len(published) > 0 {
return published, nil
}
var had []string
for key, m := range held {
if m.Inputs2 != nil && m.Inputs2.ImportForm != "" { // not Packages: a version line holds those too, with nothing published
had = append(had, key)
}
}
if len(had) > 0 {
slices.Sort(had)
return nil, fmt.Errorf("the packages state answers no published package, though %d held modules (%s ...) were computed from published ones", len(had), had[0])
}
return published, nil
}
// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
// reported.
func reached(held map[string]meshcatalog.ModulesByModuleValue) (modules, imports map[string][]string) {
modules, imports = map[string][]string{}, map[string][]string{}
add := func(to map[string][]string, p string) {
repo, _, ok := trunk.PackageRepository(p)
if ok && !slices.Contains(to[repo], p) {
to[repo] = append(to[repo], p)
}
}
for _, m := range held {
if m.Inputs2 == nil {
continue
}
for _, p := range m.Inputs2.Packages {
add(modules, p)
}
for _, p := range m.Inputs2.Imports {
add(imports, p)
}
}
for _, by := range []map[string][]string{modules, imports} {
for r := range by {
slices.Sort(by[r])
}
}
return modules, imports
}
// packagesAt are the Go modules of repository at commit that hold modules (module paths) or imports (import paths),
// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
// directory by directory). A module the listing has no go.mod for is said and left out.
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, error) {
out, _, err := ix.packagesIn(ctx, repository, commit, modules, imports)
return out, err
}
// packagesIn is packagesAt, with the modules whose directory the tree at the commit does not hold (gone), by module path.
func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, []string, error) {
if ix.Files == nil || commit == "" {
return nil, nil, nil
}
dirOf := map[string]string{} // module path to its directory in the repository
for _, m := range modules {
if _, dir, ok := trunk.PackageRepository(m); ok {
dirOf[m] = dir
}
}
holds := func(imp string) bool {
for m := range dirOf {
if imp == m || strings.HasPrefix(imp, m+"/") {
return true
}
}
return false
}
listings := map[string]map[string]string{} // by directory
list := func(dirs []string) error {
var ask []string
for _, d := range dirs {
if _, done := listings[d]; !done {
ask = append(ask, d)
}
}
if len(ask) == 0 {
return nil
}
got, err := ix.Files(ctx, repository, commit, ask)
if err != nil {
return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err)
}
for _, d := range ask {
listings[d] = got[d]
}
return nil
}
for _, imp := range imports {
if holds(imp) {
continue
}
_, dir, _ := trunk.PackageRepository(imp)
for d := dir; ; d = parent(d) {
if err := list([]string{d}); err != nil {
return nil, nil, err
}
if _, ok := listings[d]["go.mod"]; ok {
m := strings.TrimSuffix(trunk.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
dirOf[m] = d
break
}
if d == "" {
ix.logf("%s at %s: no go.mod holds the import %s, so it has no digest", repository, commit, imp)
break
}
}
}
// The modules are read in one call; when the tree at the commit lacks a directory of them (a module published at an earlier
// merge that this one removed or moved), each is read alone and the missing one is left out, said, so it never keeps the
// others from being published (issue 1585): a revision's own new module is published at the revision's own merge.
absent := map[string]bool{}
dirs := slices.Sorted(maps.Values(dirOf))
if err := list(dirs); err != nil {
if !errors.Is(err, ErrNotInTree) || len(dirs) < 2 {
return nil, nil, err
}
whole := err
for _, d := range dirs {
if err := list([]string{d}); err != nil {
if !errors.Is(err, ErrNotInTree) {
return nil, nil, err
}
ix.logf("%s at %s: the directory %q is not in the tree, so the module there is not read: %v", repository, commit, d, err)
absent[d] = true
}
}
if len(absent) == len(dirs) {
return nil, nil, whole // the tree itself fails
}
}
out := map[string]meshcatalog.PackagesByModuleValue{}
var gone []string
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
dir := dirOf[m]
if absent[dir] {
gone = append(gone, m)
continue
}
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
if err != nil {
ix.logf("%s at %s: %v; it is not published", repository, commit, err)
continue
}
d := dir
if d == "" {
d = meshcatalogclient.RootPath
}
out[m] = meshcatalog.PackagesByModuleValue{Module: m, Repository: trunk.RepositoryName(repository), Dir: d, Commit: commit,
Requires: requires, Packages: pkgs}
}
return out, gone, nil
}
// parent is the directory above d, "" for the repository's root.
func parent(d string) string {
if d == "" {
return ""
}
p := path.Dir(d)
if p == "." {
return ""
}
return p
}
// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool) error {
if ix.Packages == nil || ix.Files == nil {
return nil
}
modules, imports := reached(held)
repo := trunk.RepositoryName(repository)
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
return nil
}
have := ix.publishedAll(ctx)
if onlyMissing {
// Imports a held module already resolves are not asked again.
var ms, is []string
for _, m := range modules[repo] {
if _, ok := have[m]; !ok {
ms = append(ms, m)
}
}
for _, imp := range imports[repo] {
if len(fingerprint.Resolve(repo, []string{imp}, have).Missing) > 0 {
is = append(is, imp)
}
}
if len(ms) == 0 && len(is) == 0 {
return nil
}
modules[repo], imports[repo] = ms, is
}
// An import a published module of the repository already holds is resolved by its module's directory.
for m, p := range have {
if trunk.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
modules[repo] = append(modules[repo], m)
}
}
got, gone, err := ix.packagesIn(ctx, repo, commit, modules[repo], imports[repo])
if err != nil {
ix.notPublished(ctx, repo, commit, err)
return err
}
ix.published(ctx, repo, commit)
// A repository's packages are those in the merge's tree (design 47 H H-A-232): the kept key of a package whose directory
// the trunk's merge no longer holds is dropped, said, so the forge is never asked for it again.
for _, m := range gone {
if _, kept := have[m]; !kept {
continue
}
key, err := PackagesKey(m)
if err != nil {
ix.logf("%s: %v", m, err)
continue
}
if err := write(ctx, ix.Packages, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
return fmt.Errorf("packages.%s: %w", key, err)
}
ix.logf("package %s of %s: its directory is gone at %s: its kept key dropped", m, repo, commit)
}
for _, m := range slices.Sorted(maps.Keys(got)) {
next := got[m]
key, err := PackagesKey(m)
if err != nil {
ix.logf("%s: %v", m, err)
continue
}
err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) {
return nil, false, nil
}
}
next.At = ptr(ix.now().UTC())
data, err := EncodePublishedPackages(next)
return data, false, err
})
if err != nil {
return fmt.Errorf("packages.%s: %w", key, err)
}
}
return nil
}
// PackagesNotPublishedKind is the finding of a repository whose packages a take-in's publish could not list, for a reason
// that is not a gone directory (design 47 H H-A-233, H-A-234): scope merge, id the repository, with the forge's words;
// raised at once and cleared at the repository's next publish that lists.
const PackagesNotPublishedKind = "packages-not-published"
// notPublished raises the finding packages-not-published of repository.
func (ix *Indexer) notPublished(ctx context.Context, repository, commit string, cause error) {
if ix.Findings == nil {
return
}
f := ix.observed(mcc.ScopeMerge, repository, PackagesNotPublishedKind,
fmt.Sprintf("the packages of %s are not published at %.12s", repository, commit),
"A repository's packages are not published",
fmt.Sprintf("The take-in of %s at %s could not list its packages (%v), so their digests stay at their last published commit and a change to one does not move its importers.", repository, commit, cause),
"indexer."+PackagesNotPublishedKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("%s: the finding %s is not written: %v", repository, PackagesNotPublishedKind, err)
}
}
// published clears the finding packages-not-published of repository: its publish listed.
func (ix *Indexer) published(ctx context.Context, repository, commit string) {
if ix.Findings == nil {
return
}
if err := ix.clearFinding(ctx, mcc.ScopeMerge, repository, PackagesNotPublishedKind, "its packages are listed at "+commit); err != nil {
ix.logf("%s: the finding %s is not cleared: %v", repository, PackagesNotPublishedKind, err)
}
}
// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
// A repository with no taken merge is left for its next merge.
func (ix *Indexer) FillPackages(ctx context.Context) error {
if ix.Packages == nil || ix.Files == nil || ix.Modules == nil || ix.Taken == nil {
return nil
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
modules, imports := reached(held)
repos := map[string]bool{}
for r := range modules {
repos[r] = true
}
for r := range imports {
repos[r] = true
}
for _, repo := range slices.Sorted(maps.Keys(repos)) {
key, err := trunk.RepositoryKey(repo)
if err != nil {
continue
}
value, _, found, err := ix.Taken.Get(ctx, key)
if err != nil || !found {
continue
}
t, err := takein.DecodeTaken(value)
if err != nil {
continue
}
began := time.Now()
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
}
ix.logf("publishing the packages at start: %s at %.12s took %s", repo, t.Merge, time.Since(began).Round(time.Millisecond))
}
return nil
}
// ErrNotInTree is a failure of the tree itself: a directory asked of the forge is not in the tree at that commit. No retry
// changes it, so a head that fails so is asked once (candidatePublished).
var ErrNotInTree = errors.New("the directory is not in the tree at that commit")