mesh-delivery/modules/mesh-delivery-indexer/internal/index/modules.go

package index

import (
	"context"
	"fmt"
	"maps"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that
// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest,
// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as
// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is
// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain.
//
// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue
// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
	digests, err := ix.keepManifests(ctx, f)
	if err != nil {
		return err
	}
	held, err := ix.heldModules(ctx)
	if err != nil {
		return err
	}
	objects, namedErr := ix.namedObjects(ctx, f)
	overlay := map[string]delivery.Module{}
	dirOf := map[string]string{}
	named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
	for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
		raw := f.Manifests[dir]
		name := manifestModule(raw)
		if name == "" {
			ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir)
			every = false
			continue
		}
		key, err := delivery.ModulesKey(name)
		if err != nil {
			every = false
			ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err)
			continue
		}
		named[key] = true
		cur, has := held[key]
		if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
			ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
			continue
		}
		if other, dup := dirOf[name]; dup {
			if !has || cur.Path != pathOf(dir) {
				ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other)
				continue
			}
			ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir)
		}
		dirOf[name] = dir
		next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)}
		if next.Manifest == "" {
			next.None2 = "the manifest cannot be read: it cannot be kept as a value"
		} else {
			in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx))
			next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
			if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
				// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
				// that reads it judges the module again at start with no merge (issue 1923).
				next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir])
			}
			if in.Kept == nil && namedErr[dir] != "" {
				next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
			}
		}
		overlay[key] = next
	}
	// A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key
	// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
	if len(f.Manifests) > 0 && every {
		if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
			return err
		}
	}
	// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
	return ix.recomputeWith(ctx, overlay)
}

// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
	return func(module string) (string, string) {
		key, err := delivery.ModulesKey(module)
		if err != nil {
			return "", err.Error()
		}
		value, _, found, err := ix.Modules.Get(ctx, key)
		if err != nil {
			return "", "its record cannot be read: " + err.Error()
		}
		if !found {
			return "", "the indexer holds no record of it"
		}
		cur, err := delivery.DecodeModule(value)
		switch {
		case err != nil:
			return "", "its record cannot be read: " + err.Error()
		case cur.FP2 != "":
			return cur.FP2, ""
		case cur.None2 != "":
			return "", cur.None2
		}
		return "", "its record carries no fingerprint of the new form"
	}
}

// pathOf is the module's directory as Module.Path holds it: "." for the repository's root, since empty means not known.
func pathOf(dir string) string {
	if dir == "" {
		return delivery.RootPath
	}
	return dir
}

func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }

// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
	objects := map[string]map[string]string{}
	need := map[string][]string{}
	for dir, raw := range f.Manifests {
		m, err := catalogue.ParseManifestForInputs([]byte(raw))
		if err != nil {
			continue // inputsOfNamed says why
		}
		paths, every := m.NamedPaths()
		if !every {
			continue
		}
		if carried := f.Paths[dir]; len(carried) > 0 {
			objects[dir] = carried
			continue
		}
		need[dir] = paths
	}
	if len(need) == 0 || ix.Named == nil {
		return objects, nil
	}
	got, err := ix.Named(ctx, f.Repository, f.Merge, need)
	if err != nil {
		words := map[string]string{}
		for dir := range need {
			words[dir] = err.Error()
		}
		ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err)
		return objects, words
	}
	for dir := range need {
		objects[dir] = got[dir]
	}
	return objects, nil
}

// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
	for _, key := range slices.Sorted(maps.Keys(held)) {
		cur := held[key]
		if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
			continue
		}
		err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
			if !found {
				return nil, false, nil
			}
			if m, err := delivery.DecodeModule(value); err != nil || !delivery.SameRepository(m.Repository, repository) {
				return nil, false, nil
			}
			return nil, true, nil
		})
		if err != nil {
			return fmt.Errorf("modules.%s: %w", key, err)
		}
		ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
	}
	return nil
}

// Reconcile deletes the modules keys whose module the trunk does not hold, for each repository the latest source names a
// head of: the module directories at that head are asked of the forge (Trunk), and a module of the repository whose
// directory is not among them goes (issue 1900). It is run at the indexer's start; a merge does the same from its own
// message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is
// and said: a failure never empties the bucket.
func (ix *Indexer) Reconcile(ctx context.Context) error {
	if ix.Modules == nil || ix.Trunk == nil {
		return nil
	}
	ix.mu.Lock()
	heads := maps.Clone(ix.source.heads)
	ix.mu.Unlock()
	held, err := ix.heldModules(ctx)
	if err != nil {
		return err
	}
	for _, repository := range slices.Sorted(maps.Keys(heads)) {
		mine := map[string]delivery.Module{}
		for key, m := range held {
			if m.Repository != "" && delivery.SameRepository(m.Repository, repository) {
				mine[key] = m
			}
		}
		if len(mine) == 0 {
			continue
		}
		dirs, err := ix.Trunk(ctx, repository, heads[repository])
		if err != nil {
			ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err)
			continue
		}
		if len(dirs) == 0 {
			ix.logf("%s at %s: the trunk reads as holding no module, so none is dropped", repository, heads[repository])
			continue
		}
		there := map[string]bool{}
		for _, d := range dirs {
			there[pathOf(strings.Trim(d, "/"))] = true
		}
		head := heads[repository]
		// The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard
		// first) is not read as removed, only one whose commit the head is, or contains.
		stays := func(_ string, m delivery.Module) bool {
			if m.Path == "" || there[m.Path] {
				return true
			}
			if m.Commit == head {
				return false
			}
			if m.Commit == "" || ix.Contains == nil {
				return true
			}
			in, err := ix.Contains(ctx, repository, m.Commit, head)
			if err != nil {
				ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err)
				return true
			}
			return !in
		}
		if err := ix.dropRemoved(ctx, repository, mine, stays); err != nil {
			return err
		}
	}
	return nil
}

// None2Unreadable begins the reason of a module whose manifest the running SDK cannot read.
const None2Unreadable = "the manifest cannot be read: "

// unreadInputs are the inputs a merge carried for a module whose manifest cannot be read: the directory's tree, the lock
// hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named
// paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into
// the inputs of the module (issue 1923).
func unreadInputs(tree string, locks, paths map[string]string) *delivery.KeptInputs2 {
	if tree == "" {
		return nil
	}
	in := &delivery.KeptInputs2{Tree: tree}
	if len(locks) > 0 {
		in.Locks = maps.Clone(locks)
	}
	if len(paths) > 0 {
		in.Paths = maps.Clone(paths)
	}
	return in
}