mesh-delivery/modules/mesh-delivery-indexer/internal/index/previews.go

package index

import (
	"context"
	"fmt"
	"maps"
	"reflect"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/delivery"

	"mesh-delivery-indexer/internal/fingerprint"
)

// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull
// request's head merged onto its trunk's head, with the tree, the manifest digest and the lock hashes of each module
// directory. From one the indexer gives the candidate's touched module its new-form fingerprint, by the rules a module of
// the trunk is given its own (Recompute), or says why it has none.

// The reasons a candidate's touched module has no fingerprint of the new form while a preview exists or is missing.
const (
	// NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now.
	NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made"
	// NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files
	// in conflict follow.
	NonePreviewConflict = "does not merge: the head conflicts with the trunk in "
	// NonePreviewNoModule is for a module whose directory the preview names no manifest for: mesh-sdk's words, which the
	// assigner reads as the change removing the module (issue 1297).
	NonePreviewNoModule = delivery.CandidateNoneNoModule
	// NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold.
	NonePreviewNoManifest = "the manifest of the merge is not kept"
	// NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells.
	NoneNoDirectory = "the module's directory in its repository is not known"
)

// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key
// delivery.PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a
// preview is said in the log and left out.
func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool {
	if _, _, _, err := delivery.ParsePreviewKey(key); err != nil {
		ix.logf("previews.%s is not a preview key, left out: %v", key, err)
		return false
	}
	ix.mu.Lock()
	defer ix.mu.Unlock()
	if deleted {
		if _, had := ix.previews[key]; !had {
			return false
		}
		delete(ix.previews, key)
		return true
	}
	pv, err := delivery.DecodePreview(value)
	if err != nil {
		ix.logf("previews.%s is not a preview, left out: %v", key, err)
		return false
	}
	if ix.previews == nil {
		ix.previews = map[string]delivery.Preview{}
	}
	if cur, had := ix.previews[key]; had && delivery.SamePreview(cur, pv) {
		return false
	}
	ix.previews[key] = pv
	return true
}

// ApplyPreview takes one change of the previews in (PutPreview) and, when the previews held changed, writes the
// candidates' fingerprints again (RecomputeCandidates).
func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, deleted bool) error {
	if !ix.PutPreview(key, value, deleted) {
		return nil
	}
	return ix.RecomputeCandidates(ctx)
}

// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
// what it holds. A candidate is written only when it differs, by compare-and-set on the revision read.
func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
	if ix.Store == nil {
		return nil
	}
	ix.mu.Lock()
	none := len(ix.previews) == 0
	ix.mu.Unlock()
	if none {
		return nil // no preview held: every candidate has what the pull messages gave it
	}
	keys, err := ix.Store.Keys(ctx)
	if err != nil {
		return fmt.Errorf("listing the candidates: %w", err)
	}
	for _, key := range keys {
		err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
			if !found {
				return nil, false, nil
			}
			c, err := delivery.DecodeCandidate(value)
			if err != nil {
				ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
				return nil, false, nil
			}
			if !ix.withPreviews(ctx, &c) {
				return nil, false, nil
			}
			data, err := delivery.EncodeCandidate(c)
			return data, false, err
		})
		if err != nil {
			return fmt.Errorf("candidates.%s: %w", key, err)
		}
	}
	return nil
}

// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
	ix.mu.Lock()
	defer ix.mu.Unlock()
	trunk := heads[repository]
	if trunk == "" {
		return delivery.Preview{}, false, false
	}
	if key, err := delivery.PreviewKey(repository, head, trunk); err == nil {
		if pv, ok := ix.previews[key]; ok {
			return pv, true, false
		}
	}
	for _, pv := range ix.previews {
		if pv.Repository == repository && pv.Head == head {
			return delivery.Preview{}, false, true
		}
	}
	return delivery.Preview{}, false, false
}

// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
// no preview keeps what it had.
//
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
// fingerprint first appeared at.
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
	ix.mu.Lock()
	rev, heads := ix.source.rev, ix.source.heads
	ix.mu.Unlock()
	changed, read := false, false
	// A package one of the candidate's pull requests publishes is built at that pull request's head, every other at its
	// latest published version (novox/hq ADR 0573 §4): a group's members build against the group's own package.
	members := map[string]string{}
	for _, p := range c.Pulls {
		members[delivery.RepositoryName(p.Repository)] = p.Head.Head
	}
	var held map[string]delivery.Module
	listed := false
	for i := range c.Pulls {
		p := &c.Pulls[i]
		pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
		if !found && !stale {
			continue
		}
		read = true
		if !listed {
			var err error
			if held, err = ix.heldModules(ctx); err != nil {
				ix.logf("the modules cannot be listed, so no touched module's directory or package version is known: %v", err)
			}
			listed = true
		}
		// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
		// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
		dirs := map[string]string{}
		for name, m := range held {
			if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" {
				dirs[name] = m.Path
			}
		}
		for _, m := range p.Modules {
			if _, known := dirs[m]; known || p.Fps[m].Manifest == "" {
				continue
			}
			for dir, pm := range pv.Modules {
				if pm.Manifest == p.Fps[m].Manifest {
					dirs[m] = dir
				}
			}
		}
		fps := maps.Clone(p.Fps)
		for _, m := range p.Modules {
			next := delivery.CandidateModule{Manifest: p.Fps[m].Manifest}
			switch {
			case stale:
				next.None2 = NonePreviewStale
			case len(pv.Conflicts) > 0:
				next.None2 = NonePreviewConflict + strings.Join(pv.Conflicts, ", ")
			default:
				dir, known := dirs[m]
				if !known {
					next.None2 = NoneNoDirectory
					break
				}
				next = ix.candidateModule(ctx, m, dir, pv, held, members)
			}
			if fps == nil {
				fps = map[string]delivery.CandidateModule{}
			}
			if !reflect.DeepEqual(fps[m], next) {
				fps[m] = next
				changed = true
			}
		}
		if changed || !maps.EqualFunc(fps, p.Fps, func(a, b delivery.CandidateModule) bool { return reflect.DeepEqual(a, b) }) {
			p.Fps = fps
		}
	}
	if changed {
		c.FPSet = delivery.FingerprintSet(c.Pulls)
	}
	if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
		c.Base, changed = rev, true
	}
	return changed
}

// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, the toolchains
// the build modules agree on, and the version of each mesh package it uses (candidatePackages), by the rules Recompute
// applies to a module of the trunk.
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview, held map[string]delivery.Module, members map[string]string) delivery.CandidateModule {
	pm, ok := pv.Modules[dir]
	if !ok && dir == "" {
		pm, ok = pv.Modules[delivery.RootPath]
	}
	if !ok && dir == delivery.RootPath {
		pm, ok = pv.Modules[""]
	}
	if !ok {
		return delivery.CandidateModule{None2: NonePreviewNoModule}
	}
	out := delivery.CandidateModule{Manifest: pm.Manifest}
	if ix.Manifests == nil {
		out.None2 = NonePreviewNoManifest
		return out
	}
	value, _, found, err := ix.Manifests.Get(ctx, pm.Manifest)
	if err != nil || !found {
		out.None2 = NonePreviewNoManifest
		return out
	}
	kept, err := delivery.DecodeManifest(value)
	if err != nil {
		out.None2 = NonePreviewNoManifest
		return out
	}
	// The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer
	// computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build
	// module still names them.
	trunk := held[m]
	packages := fingerprint.OtherPackages(pm.Packages, pv.Repository)
	if len(pm.Packages) == 0 && trunk.Inputs2 != nil {
		packages = trunk.Inputs2.Packages
	}
	in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
	if in.Kept == nil {
		out.None2 = in.None2
		return out
	}
	if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Pending {
		out.None2 = in.None2 // a context, or a base with no fingerprint
		return out
	}
	tools := ix.agreed()
	toolchains := map[string]delivery.Toolchain2{}
	for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) {
		t := tools[in.Languages[artifact]]
		if t.none != "" {
			out.None2 = t.none
			return out
		}
		toolchains[artifact] = t.tc
	}
	inputs := *in.Kept // no builder input (ADR 0530 §1)
	var fp, none string
	var versions delivery.PackageVersions
	if trunk.Inputs2 != nil && len(trunk.Inputs2.Imports) > 0 {
		inputs.Imports, inputs.ImportForm = trunk.Inputs2.Imports, trunk.Inputs2.ImportForm
	}
	// The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk
	// computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed.
	published := ix.candidatePublished(ctx, members, held)
	if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 {
		f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, nil)
		fp, none, versions = f.fp, f.none, f.versions
	} else {
		inputs.Imports, inputs.ImportForm = nil, ""
		versions = candidatePackages(inputs.Packages, held, members)
		in := inputsFrom(m, inputs, toolchains)
		if len(inputs.Packages) > 0 {
			in.Packages = map[string]string{}
			for _, p := range inputs.Packages {
				in.Packages[p] = versions[p]
			}
		}
		fp, none = fingerprint.Of(in)
	}
	if fp == "" {
		out.None2 = none
		return out
	}
	out.FP2, out.Packages = fp, versions
	if len(inputs.Paths) > 0 {
		out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
	}
	return out
}

// candidatePackages is the version each package of a candidate's module is built against (novox/hq ADR 0573 §2, §4): the
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
// version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none.
func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions {
	versions, _ := fingerprint.HeldVersionsOf(packages, held)
	for _, p := range packages {
		repository, _, _ := delivery.PackageRepository(p)
		if head := members[repository]; head != "" {
			if versions == nil {
				versions = delivery.PackageVersions{}
			}
			versions[p] = head
		}
	}
	return versions
}