mesh-delivery/modules/mesh-delivery-indexer/internal/index/modules.go

package index

import (
	"context"
	"fmt"
	"maps"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/delivery"

	"mesh-delivery-indexer/judge"
)

// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that
// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest,
// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as
// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is
// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain.
//
// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue
// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7), and kept waiting: once the holding repository's trunk drops the module, the merge that drops it hands it to the
// waiting repository instead of deleting it (moves.go). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
//
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
// deletes nothing, and is said; it is asked once, before anything is written.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) {
	return ix.applyManifests(ctx, f, true)
}

// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
	return ix.applyMoving(ctx, f, drop, nil)
}

// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
	w := newMergeWrite(f)
	if drop {
		if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
			return w, err
		}
	}
	digests, err := ix.keepManifests(ctx, f)
	if err != nil {
		return w, err
	}
	held, err := ix.heldModules(ctx)
	if err != nil {
		return w, err
	}
	objects, namedErr := ix.namedObjects(ctx, f)
	overlay := map[string]delivery.Module{}
	dirOf := map[string]string{}
	named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
	for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
		raw := f.Manifests[dir]
		name := manifestModule(raw)
		if name == "" {
			ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir)
			every = false
			continue
		}
		key, err := delivery.ModulesKey(name)
		if err != nil {
			every = false
			ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err)
			continue
		}
		named[key] = true
		cur, has := held[key]
		if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
			if !moves[key] {
				ix.waitToMove(key, f, dir)
				ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository)
				continue
			}
			ix.logf("%s at %s: the module %s moves here from %s, whose trunk no longer holds it", f.Repository, f.Merge, name, cur.Repository)
		}
		if other, dup := dirOf[name]; dup {
			if !has || cur.Path != pathOf(dir) {
				ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other)
				continue
			}
			ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir)
		}
		dirOf[name] = dir
		next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)}
		if next.Manifest == "" {
			next.None2 = "the manifest cannot be read: it cannot be kept as a value"
		} else {
			packages := delivery.OtherPackages(f.Packages[dir], f.Repository) // its own repository's come with its tree
			in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
			next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
			if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
				// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
				// that reads it judges the module again at start with no merge (issue 1923).
				next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir], packages)
			}
			if in.Kept == nil && namedErr[dir] != "" {
				next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
			}
		}
		overlay[key] = next
	}
	// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
	// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
	if drop && len(f.Manifests) > 0 && every {
		if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
			return w, err
		}
	}
	// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
	return w, ix.recomputeWith(ctx, overlay, w)
}

// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
	return func(module string) (string, string) {
		key, err := delivery.ModulesKey(module)
		if err != nil {
			return "", err.Error()
		}
		value, _, found, err := ix.Modules.Get(ctx, key)
		if err != nil {
			return "", "its record cannot be read: " + err.Error()
		}
		if !found {
			return "", "the indexer holds no record of it"
		}
		cur, err := delivery.DecodeModule(value)
		switch {
		case err != nil:
			return "", "its record cannot be read: " + err.Error()
		case cur.FP2 != "":
			return cur.FP2, ""
		case cur.None2 != "":
			return "", cur.None2
		}
		return "", "its record carries no fingerprint of the new form"
	}
}

// pathOf is the module's directory as Module.Path holds it: "." for the repository's root, since empty means not known.
func pathOf(dir string) string {
	if dir == "" {
		return delivery.RootPath
	}
	return dir
}

func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }

// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
	objects := map[string]map[string]string{}
	need := map[string][]string{}
	for dir, raw := range f.Manifests {
		m, err := judge.ForInputs([]byte(raw))
		if err != nil {
			continue // inputsOfNamed says why
		}
		paths, every := m.NamedPaths()
		if !every {
			continue
		}
		if carried := f.Paths[dir]; len(carried) > 0 {
			objects[dir] = carried
			continue
		}
		need[dir] = paths
	}
	if len(need) == 0 || ix.Named == nil {
		return objects, nil
	}
	got, err := ix.Named(ctx, f.Repository, f.Merge, need)
	if err != nil {
		words := map[string]string{}
		for dir := range need {
			words[dir] = err.Error()
		}
		ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err)
		return objects, words
	}
	for dir := range need {
		objects[dir] = got[dir]
	}
	return objects, nil
}

// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
	var handed []string
	for _, key := range slices.Sorted(maps.Keys(held)) {
		cur := held[key]
		if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
			continue
		}
		if ix.movingTo(key, repository) {
			handed = append(handed, key) // written in place by the repository it moves to, never deleted first
			continue
		}
		err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
			if !found {
				return nil, false, nil
			}
			if m, err := delivery.DecodeModule(value); err != nil || !delivery.SameRepository(m.Repository, repository) {
				return nil, false, nil
			}
			return nil, true, nil
		})
		if err != nil {
			return fmt.Errorf("modules.%s: %w", key, err)
		}
		ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
	}
	for _, key := range handed {
		if err := ix.handOver(ctx, key); err != nil {
			return err
		}
	}
	return nil
}

// Reconcile deletes the modules keys whose module the trunk does not hold, for each repository the latest source names a
// head of: the module directories at that head are asked of the forge (Trunk), and a module of the repository whose
// directory is not among them goes (issue 1900). It is run at the indexer's start; a merge does the same from its own
// message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is
// and said: a failure never empties the bucket.
func (ix *Indexer) Reconcile(ctx context.Context) error {
	if ix.Modules == nil || ix.Trunk == nil {
		return nil
	}
	ix.mu.Lock()
	heads := maps.Clone(ix.source.heads)
	ix.mu.Unlock()
	held, err := ix.heldModules(ctx)
	if err != nil {
		return err
	}
	for _, repository := range slices.Sorted(maps.Keys(heads)) {
		mine := map[string]delivery.Module{}
		for key, m := range held {
			if m.Repository != "" && delivery.SameRepository(m.Repository, repository) {
				mine[key] = m
			}
		}
		if len(mine) == 0 {
			continue
		}
		dirs, err := ix.Trunk(ctx, repository, heads[repository])
		if err != nil {
			ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err)
			continue
		}
		if len(dirs) == 0 {
			ix.logf("%s at %s: the trunk reads as holding no module, so none is dropped", repository, heads[repository])
			continue
		}
		there := map[string]bool{}
		for _, d := range dirs {
			there[pathOf(strings.Trim(d, "/"))] = true
		}
		head := heads[repository]
		// The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard
		// first) is not read as removed, only one whose commit the head is, or contains.
		stays := func(_ string, m delivery.Module) bool {
			if m.Path == "" || there[m.Path] {
				return true
			}
			if m.Commit == head {
				return false
			}
			if m.Commit == "" || ix.Contains == nil {
				return true
			}
			in, err := ix.Contains(ctx, repository, m.Commit, head)
			if err != nil {
				ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err)
				return true
			}
			return !in
		}
		if err := ix.dropRemoved(ctx, repository, mine, stays); err != nil {
			return err
		}
	}
	return nil
}

// None2Unreadable begins the reason of a module whose manifest the running SDK cannot read.
const None2Unreadable = "the manifest cannot be read: "

// unreadInputs are the inputs a merge carried for a module whose manifest cannot be read: the directory's tree, the lock
// hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named
// paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into
// the inputs of the module (issue 1923).
func unreadInputs(tree string, locks, paths map[string]string, packages []string) *delivery.KeptInputs2 {
	if tree == "" {
		return nil
	}
	in := &delivery.KeptInputs2{Tree: tree}
	if len(locks) > 0 {
		in.Locks = maps.Clone(locks)
	}
	if len(paths) > 0 {
		in.Paths = maps.Clone(paths)
	}
	in.Packages = slices.Clone(packages)
	return in
}