mesh-delivery/modules/mesh-delivery-indexer/internal/index/packages.go

package index

import (
	"context"
	"fmt"
	"maps"
	"path"
	"reflect"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/delivery"

	"mesh-delivery-indexer/internal/fingerprint"
)

// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
// digests in its state `packages` (delivery.PackagesBucket), one key per Go module. The builder, the checker and the
// build modules read it there; nobody else computes a digest or a version.
//
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
// reported (KeptInputs2.Imports). Its key is written at each merge of its repository, before the merge's recompute, so
// a merge that changes a package moves its importers in the same recompute (§3); and once at the start, for each
// reached module the state does not hold yet (FillPackages).

// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.PublishedPackages {
	out := map[string]delivery.PublishedPackages{}
	if ix.Packages == nil {
		return out
	}
	keys, err := ix.Packages.Keys(ctx)
	if err != nil {
		ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err)
		return out
	}
	for _, key := range keys {
		value, _, found, err := ix.Packages.Get(ctx, key)
		if err != nil || !found {
			continue
		}
		p, err := delivery.DecodePublishedPackages(value)
		if err != nil {
			ix.logf("packages.%s holds what is not a published module, left out: %v", key, err)
			continue
		}
		out[p.Module] = p
	}
	return out
}

// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
// reported.
func reached(held map[string]delivery.Module) (modules, imports map[string][]string) {
	modules, imports = map[string][]string{}, map[string][]string{}
	add := func(to map[string][]string, p string) {
		repo, _, ok := delivery.PackageRepository(p)
		if ok && !slices.Contains(to[repo], p) {
			to[repo] = append(to[repo], p)
		}
	}
	for _, m := range held {
		if m.Inputs2 == nil {
			continue
		}
		for _, p := range m.Inputs2.Packages {
			add(modules, p)
		}
		for _, p := range m.Inputs2.Imports {
			add(imports, p)
		}
	}
	for _, by := range []map[string][]string{modules, imports} {
		for r := range by {
			slices.Sort(by[r])
		}
	}
	return modules, imports
}

// packagesAt are the Go modules of repository at commit that hold modules (module paths) or imports (import paths),
// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
// directory by directory). A module the listing has no go.mod for is said and left out.
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]delivery.PublishedPackages, error) {
	if ix.Files == nil || commit == "" {
		return nil, nil
	}
	dirOf := map[string]string{} // module path to its directory in the repository
	for _, m := range modules {
		if _, dir, ok := delivery.PackageRepository(m); ok {
			dirOf[m] = dir
		}
	}
	holds := func(imp string) bool {
		for m := range dirOf {
			if imp == m || strings.HasPrefix(imp, m+"/") {
				return true
			}
		}
		return false
	}
	listings := map[string]map[string]string{} // by directory
	list := func(dirs []string) error {
		var ask []string
		for _, d := range dirs {
			if _, done := listings[d]; !done {
				ask = append(ask, d)
			}
		}
		if len(ask) == 0 {
			return nil
		}
		got, err := ix.Files(ctx, repository, commit, ask)
		if err != nil {
			return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err)
		}
		for _, d := range ask {
			listings[d] = got[d]
		}
		return nil
	}
	for _, imp := range imports {
		if holds(imp) {
			continue
		}
		_, dir, _ := delivery.PackageRepository(imp)
		for d := dir; ; d = parent(d) {
			if err := list([]string{d}); err != nil {
				return nil, err
			}
			if _, ok := listings[d]["go.mod"]; ok {
				m := strings.TrimSuffix(delivery.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
				dirOf[m] = d
				break
			}
			if d == "" {
				ix.logf("%s at %s: no go.mod holds the import %s, so it has no digest", repository, commit, imp)
				break
			}
		}
	}
	if err := list(slices.Sorted(maps.Values(dirOf))); err != nil {
		return nil, err
	}
	out := map[string]delivery.PublishedPackages{}
	for _, m := range slices.Sorted(maps.Keys(dirOf)) {
		dir := dirOf[m]
		requires, pkgs, err := fingerprint.Packages(m, listings[dir])
		if err != nil {
			ix.logf("%s at %s: %v; it is not published", repository, commit, err)
			continue
		}
		d := dir
		if d == "" {
			d = delivery.RootPath
		}
		out[m] = delivery.PublishedPackages{Module: m, Repository: delivery.RepositoryName(repository), Dir: d, Commit: commit,
			Requires: requires, Packages: pkgs}
	}
	return out, nil
}

// parent is the directory above d, "" for the repository's root.
func parent(d string) string {
	if d == "" {
		return ""
	}
	p := path.Dir(d)
	if p == "." {
		return ""
	}
	return p
}

// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]delivery.Module, onlyMissing bool) error {
	if ix.Packages == nil || ix.Files == nil {
		return nil
	}
	modules, imports := reached(held)
	repo := delivery.RepositoryName(repository)
	if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
		return nil
	}
	have := ix.publishedAll(ctx)
	if onlyMissing {
		// Imports a held module already resolves are not asked again.
		var ms, is []string
		for _, m := range modules[repo] {
			if _, ok := have[m]; !ok {
				ms = append(ms, m)
			}
		}
		for _, imp := range imports[repo] {
			if len(fingerprint.Resolve(repo, []string{imp}, have).Missing) > 0 {
				is = append(is, imp)
			}
		}
		if len(ms) == 0 && len(is) == 0 {
			return nil
		}
		modules[repo], imports[repo] = ms, is
	}
	// An import a published module of the repository already holds is resolved by its module's directory.
	for m, p := range have {
		if delivery.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
			modules[repo] = append(modules[repo], m)
		}
	}
	got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo])
	if err != nil {
		return err
	}
	for _, m := range slices.Sorted(maps.Keys(got)) {
		next := got[m]
		key, err := delivery.PackagesKey(m)
		if err != nil {
			ix.logf("%s: %v", m, err)
			continue
		}
		err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) {
			if found {
				if cur, err := delivery.DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
					cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) {
					return nil, false, nil
				}
			}
			next.At = ix.now().UTC()
			data, err := delivery.EncodePublishedPackages(next)
			return data, false, err
		})
		if err != nil {
			return fmt.Errorf("packages.%s: %w", key, err)
		}
	}
	return nil
}

// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
// A repository with no taken merge is left for its next merge.
func (ix *Indexer) FillPackages(ctx context.Context) error {
	if ix.Packages == nil || ix.Files == nil || ix.Modules == nil || ix.Taken == nil {
		return nil
	}
	held, err := ix.heldModules(ctx)
	if err != nil {
		return err
	}
	modules, imports := reached(held)
	repos := map[string]bool{}
	for r := range modules {
		repos[r] = true
	}
	for r := range imports {
		repos[r] = true
	}
	for _, repo := range slices.Sorted(maps.Keys(repos)) {
		key, err := delivery.RepositoryKey(repo)
		if err != nil {
			continue
		}
		value, _, found, err := ix.Taken.Get(ctx, key)
		if err != nil || !found {
			continue
		}
		t, err := delivery.DecodeTaken(value)
		if err != nil {
			continue
		}
		if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
			ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
		}
	}
	return nil
}

// candidatePublished is the packages state as a candidate builds against it (novox/hq ADR 0573 §4): each Go module a
// member's repository publishes at the member's head (members, by repository), every other at its latest published
// version. A head's modules are read from the forge once and kept while the head is a member somewhere; one the forge
// cannot list is said and left at its published version.
func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]delivery.Module) map[string]delivery.PublishedPackages {
	out := ix.publishedAll(ctx)
	if len(members) == 0 {
		return out
	}
	modules, imports := reached(held)
	for _, repo := range slices.Sorted(maps.Keys(members)) {
		head := members[repo]
		if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
			continue
		}
		at := repo + "@" + head
		ix.mu.Lock()
		got, ok := ix.atHeads[at]
		ix.mu.Unlock()
		if !ok {
			var ms []string
			ms = append(ms, modules[repo]...)
			for m, p := range out {
				if delivery.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) {
					ms = append(ms, m)
				}
			}
			var err error
			if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil {
				ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
				continue
			}
			ix.mu.Lock()
			if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
				ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
			}
			ix.atHeads[at] = got
			ix.mu.Unlock()
		}
		maps.Copy(out, got)
	}
	return out
}

// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again.
const maxHeadsKept = 64