mesh-delivery/modules/mesh-delivery-indexer/internal/index/trycandidates.go

package index

// **A group try's candidate** (issue 1921; novox/hq ADR 0593 §7): a try of several repositories at exact commits has no pull
// request, so the forge bridge sends the indexer no message for it; yet the node-engine's compose-candidate and the checker's
// manifest validator read the indexer's candidate of a group. The indexer therefore watches the checker's state tries (the
// checker keeps it; this module reads it) and, for a try that is not narrowed to a level or to some tests, writes
// `candidates.<try id>`: one pull per member (number 0), its modules and manifests taken from the merge preview the try's
// fetch wrote, its manifests kept and judged with the seat definitions the members carry, as a take-in judges a revision's.
// The fingerprints come with the previews (RecomputeCandidates). The candidate is deleted when the try ends. `try-<digits>`
// is the reserved group name of a try: the bridge's open message never takes it out, and the checker never checks it.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"path"
	"slices"
	"strings"
	"time"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
	"git.novox.be/novox/mesh-sdk/go/keys"
)

// TryCandidateTools: the tools that read, drop and recover a try's candidate.
const (
	TryCandidatesTool    = "indexer_try_candidates"
	DropTryCandidateTool = "indexer_drop_try_candidate"
	RecoverTriesTool     = "indexer_recover_tries"
	// TryRecoveryState is the indexer's state of how the last recovery of the open tries went: the bucket
	// mesh-delivery-indexer_try-recovery, one key (TryRecoveryKey).
	TryRecoveryState  = "try-recovery"
	TryRecoveryBucket = Module + "_" + TryRecoveryState
	TryRecoveryKey    = "state"
)

// TryRecovery is the value of the try-recovery state: whether the candidates of the open tries were recovered at the
// indexer's start or by hand, and when it began to fail, why.
type TryRecovery struct {
	V     int       `json:"v"`
	Done  bool      `json:"done"`
	Since time.Time `json:"since,omitzero"`
	Why   string    `json:"why,omitempty"`
	At    time.Time `json:"at"`
}

// TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto.
type TryMember struct{ Repository, Head, Trunk string }

// StartTry takes a whole try in, as the checker said it started (checkermsg.Started): its members at their commits are kept, and
// its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596).
func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error {
	ix.tryMu.Lock()
	defer ix.tryMu.Unlock()
	ix.retryRecovery(ctx)
	return ix.startTry(ctx, id, members)
}

// retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds.
func (ix *Indexer) retryRecovery(ctx context.Context) {
	if ix.recoveryFailed.Load() && ix.OpenTries != nil {
		_ = ix.recoverNow(ctx)
	}
}

func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) error {
	if !meshcatalogclient.IsTryGroup(id) || len(members) == 0 {
		return fmt.Errorf("%q with %d members is no try to start", id, len(members))
	}
	ix.mu.Lock()
	if ix.tries == nil {
		ix.tries = map[string][]TryMember{}
	}
	ix.tries[id] = slices.Clone(members)
	ix.mu.Unlock()
	return ix.writeTryCandidate(ctx, id)
}

// EndTry takes a try out, as the checker said it ended (checkermsg.Ended), and deletes its candidate.
func (ix *Indexer) EndTry(ctx context.Context, id string) error {
	ix.tryMu.Lock()
	defer ix.tryMu.Unlock()
	ix.retryRecovery(ctx)
	return ix.endTry(ctx, id)
}

func (ix *Indexer) endTry(ctx context.Context, id string) error {
	ix.mu.Lock()
	delete(ix.tries, id)
	ix.mu.Unlock()
	if !meshcatalogclient.IsTryGroup(id) {
		return nil
	}
	return ix.dropTryCandidate(ctx, id)
}

// RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait
// or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them
// (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask.
//
// A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds:
// the events are still heard after it, and indexer_recover_tries runs it again by hand.
//
// A recovery and a try event never overlap (tryMu): a sweep that deleted the candidate a start event wrote after the call to the
// checker would leave a running try with none.
func (ix *Indexer) RecoverTries(ctx context.Context) error {
	ix.tryMu.Lock()
	defer ix.tryMu.Unlock()
	return ix.recoverNow(ctx)
}

func (ix *Indexer) recoverNow(ctx context.Context) error {
	if ix.OpenTries == nil || ix.Store == nil {
		return nil
	}
	err := ix.recoverTries(ctx)
	ix.recoveryFailed.Store(err != nil)
	ix.noteRecovery(ctx, err)
	return err
}

// noteRecovery writes how a recovery went: failing keeps the time it began to fail, and the journal says it each time.
func (ix *Indexer) noteRecovery(ctx context.Context, err error) {
	now := ix.now()
	r := TryRecovery{V: 1, Done: err == nil, At: now.UTC()}
	if err != nil {
		r.Why, r.Since = err.Error(), now.UTC()
		ix.logf("RECOVERY OF THE TRIES NOT DONE: %v (the try candidates may be stale or missing until indexer_recover_tries or the next start succeeds)", err)
	}
	if ix.TryRecovery == nil {
		return
	}
	for range writeTries {
		cur, rev, found, gerr := ix.TryRecovery.Get(ctx, TryRecoveryKey)
		if gerr != nil {
			ix.logf("the try-recovery state cannot be read: %v", gerr)
			return
		}
		if err != nil && found {
			var was TryRecovery
			if json.Unmarshal(cur, &was) == nil && !was.Done && !was.Since.IsZero() {
				r.Since = was.Since
			}
		}
		data, _ := json.Marshal(r)
		var werr error
		if found {
			werr = ix.TryRecovery.Update(ctx, TryRecoveryKey, data, rev)
		} else {
			werr = ix.TryRecovery.Create(ctx, TryRecoveryKey, data)
		}
		if werr == nil {
			return
		}
		if !errors.Is(werr, ErrConflict) {
			ix.logf("the try-recovery state cannot be written: %v", werr)
			return
		}
	}
}

// RecoverTriesByHand runs the recovery now (indexer_recover_tries), a hand act said in the journal.
func (ix *Indexer) RecoverTriesByHand(ctx context.Context, by, why string) TakenIn {
	r := TakenIn{Module: "tries"}
	ix.logf("recovery of the tries asked by %s (%s)", by, why)
	if ix.OpenTries == nil {
		r.Answer = "the indexer is wired to list no open try: nothing to recover from"
		return r
	}
	if err := ix.RecoverTries(ctx); err != nil {
		r.Answer = "recovery not done: " + err.Error()
		return r
	}
	r.Stored, r.Answer = true, "the candidates of the open tries are recovered"
	return r
}

func (ix *Indexer) recoverTries(ctx context.Context) error {
	open, err := ix.OpenTries(ctx)
	if err != nil {
		return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err)
	}
	live := map[string]bool{}
	for _, o := range open {
		if !o.Whole || len(o.Members) == 0 {
			continue
		}
		live[o.ID] = true
		var ms []TryMember
		for _, m := range o.Members {
			ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk})
		}
		if err := ix.startTry(ctx, o.ID, ms); err != nil {
			return err
		}
	}
	keys, err := ix.Store.Keys(ctx)
	if err != nil {
		return err
	}
	for _, k := range keys {
		if meshcatalogclient.IsTryGroup(k) && !live[k] {
			if err := ix.endTry(ctx, k); err != nil {
				return err
			}
		}
	}
	return nil
}

// WriteTryCandidates writes the candidate of every live try that has none yet and whose previews are all there: after a
// preview was written, the try that waited for it.
func (ix *Indexer) WriteTryCandidates(ctx context.Context) error {
	ix.mu.Lock()
	ids := make([]string, 0, len(ix.tries))
	for id := range ix.tries {
		ids = append(ids, id)
	}
	ix.mu.Unlock()
	slices.Sort(ids)
	for _, id := range ids {
		if err := ix.writeTryCandidate(ctx, id); err != nil {
			return err
		}
	}
	return nil
}

func (ix *Indexer) dropTryCandidate(ctx context.Context, id string) error {
	if ix.Store == nil {
		return nil
	}
	key, err := meshcatalogclient.CandidatesKey(id)
	if err != nil {
		return err
	}
	return write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil })
}

// DropTryCandidate deletes a try's candidate by hand (indexer_drop_try_candidate); a live try gets it again at its next
// change.
func (ix *Indexer) DropTryCandidate(ctx context.Context, id, by, why string) TakenIn {
	r := TakenIn{Module: id}
	switch {
	case !meshcatalogclient.IsTryGroup(id):
		r.Answer = id + " is no try's group, try-<digits>: nothing dropped"
	default:
		if err := ix.dropTryCandidate(ctx, id); err != nil {
			r.Answer = "the candidate of " + id + " could not be dropped: " + err.Error()
		} else {
			r.Stored, r.Answer = true, "the candidate of "+id+" is dropped by "+by+" ("+why+")"
			ix.logf("%s", r.Answer)
		}
	}
	return r
}

// writeTryCandidate writes the candidate of a live try when it has none at its members' heads and every member's merge
// preview is held.
func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error {
	if ix.Store == nil || ix.Read == nil {
		return nil
	}
	ix.mu.Lock()
	members := slices.Clone(ix.tries[id])
	ix.mu.Unlock()
	if len(members) == 0 {
		return nil
	}
	key, err := meshcatalogclient.CandidatesKey(id)
	if err != nil {
		return err
	}
	slices.SortFunc(members, func(a, b TryMember) int { return strings.Compare(a.Repository, b.Repository) })
	heads := make([]meshdeliveryclient.Head, len(members))
	for i, m := range members {
		heads[i] = meshdeliveryclient.Head{Repository: m.Repository, Head: m.Head}
	}
	hash, err := meshdeliveryclient.HeadsHash(heads)
	if err != nil {
		return err
	}
	if value, _, found, err := ix.Store.Get(ctx, key); err == nil && found {
		if c, err := DecodeCandidate(value); err == nil && c.Heads == hash {
			return nil // fingerprints follow the previews (RecomputeCandidates)
		}
	}
	previews := make([]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, len(members))
	for i, m := range members {
		pk, err := PreviewKey(m.Repository, m.Head, m.Trunk)
		if err != nil {
			return fmt.Errorf("the try %s: %w", id, err)
		}
		ix.mu.Lock()
		pv, ok := ix.previews[pk]
		ix.mu.Unlock()
		if !ok {
			return nil // its fetch has not written the preview yet: this is called again when it does
		}
		previews[i] = pv
	}
	held, err := ix.heldModules(ctx)
	if err != nil {
		return err
	}
	pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(members))
	var entries []PendingEntry
	over := map[string]string{}
	for i, m := range members {
		pull := meshcatalog.CandidatesByGroupValuePullsItem{Repository: heads[i].Repository, Head: heads[i].Head, Modules: []string{}}
		// defs are every module directory's manifest of the member at its head, the unchanged ones too: a contracts module whose
		// seat.json changed while its module.json did not carries definitions all the same (ADR 0636 §4).
		defs := map[string]string{}
		for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) {
			pm := previews[i].Modules[dir]
			name, unchanged := "", false
			for n, h := range held {
				if strings.EqualFold(h.Repository, m.Repository) && h.Path == forgeDir(dir) {
					name, unchanged = n, h.Manifest == pm.Manifest
					break
				}
			}
			if unchanged {
				if raw := ix.keptManifest(ctx, pm.Manifest); raw != "" {
					defs[dir] = raw
				}
				continue // the manifest the trunk holds: the try does not touch the module
			}
			rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir))
			if err != nil {
				return fmt.Errorf("the try %s: reading the manifest of %s at %.12s: %w", id, dir, m.Head, err)
			}
			raw := rr.Manifests[forgeDir(dir)]
			if raw == "" {
				ix.logf("the try %s: %s holds no manifest at %.12s, so no module of it is taken", id, dir, m.Head)
				continue
			}
			module := manifestModule(raw)
			if module == "" {
				module = name
			}
			if module == "" {
				module = path.Base(strings.Trim(dir, "/."))
			}
			digest, err := ix.keepManifest(ctx, module, raw)
			if err != nil {
				return err
			}
			if digest != pm.Manifest {
				ix.logf("the try %s: the manifest of %s read at %.12s is not the one its preview names, left", id, dir, m.Head)
				continue
			}
			if pull.Fps == nil {
				pull.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
			}
			pull.Modules = append(pull.Modules, module)
			pull.Fps[module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview}
			defs[dir] = raw
			over[module] = digest
		}
		pulls[i] = pull
		entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: defs}})
	}
	// The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4), in one pass that takes the
	// seats. Their verdicts are the verdicts state's, keyed by the manifest's digest, as a pull request's candidate's are
	// (judgeDigests in ApplyMessage): nothing reads them but the engines' composition of that digest and the module
	// value of a trunk module with the same digest.
	seats, err := ix.revisionSeats(ctx, entries)
	if err != nil {
		return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err)
	}
	ix.judgeDigestsWith(ctx, seats, over, "the try "+id)
	c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: id, Heads: hash, Pulls: pulls}
	c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
	ix.withPreviews(ctx, &c)
	if err := ix.refuseDisagreement(c); err != nil {
		return err
	}
	data, err := EncodeCandidate(c)
	if err != nil {
		return err
	}
	return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil })
}

// keptManifest is the text of the manifest the indexer keeps under a digest, or empty.
func (ix *Indexer) keptManifest(ctx context.Context, digest string) string {
	if ix.Manifests == nil || digest == "" {
		return ""
	}
	key, err := meshcatalogclient.ManifestsKey(digest)
	if err != nil {
		return ""
	}
	value, _, found, err := ix.Manifests.Get(ctx, key)
	if err != nil || !found {
		return ""
	}
	m, err := meshcatalogclient.DecodeManifest(value)
	if err != nil {
		return ""
	}
	return m.Manifest
}

func mapKeys[V any](m map[string]V) func(func(string) bool) {
	return func(yield func(string) bool) {
		for k := range m {
			if !yield(k) {
				return
			}
		}
	}
}

// forgeDir is a preview's directory as the forge reader spells it: the root as "".
func forgeDir(dir string) string {
	if dir == "." || dir == "/" {
		return ""
	}
	return strings.Trim(dir, "/")
}

// heldAt says the trunk holds a module of the repository in that directory.
func heldAt(held map[string]meshcatalog.ModulesByModuleValue, repository, dir string) bool {
	for _, h := range held {
		if strings.EqualFold(h.Repository, repository) && h.Path == forgeDir(dir) {
			return true
		}
	}
	return false
}