mesh-delivery/modules/mesh-delivery-indexer/internal/index/toolchains.go

package index

import (
	"context"
	"fmt"
	"maps"
	"reflect"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes
// (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in
// modules.<m>.Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each
// recompute reads the manifest of every module from manifests.<digest> and takes the modules in the order their bases
// and the builder come first.

const (
	// BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is
	// its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it.
	BuilderModule = "mesh-delivery-builder"
	// BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the
	// builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5).
	BuilderBuiltBy = "build-agent"
)

// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the
// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on.
var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"}

// ToolchainStreams are the stream each build module carries its toolchain.reported event on, by module, as the bus names it
// (a stream's local name in capitals, catalogue.StreamBusName): `toolchains-go` and `toolchains-node`, declared by the
// build modules (streams, with the event toolchain.reported) and heard by this module (consumes).
var ToolchainStreams = map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"}

// PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's
// toolchain of a language (key <node>.<language>, delivery.ToolchainKey), or its delete. It answers whether what the
// language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's
// language is said in the log and left out.
func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool {
	language := ""
	for l, m := range ToolchainModules {
		if m == module {
			language = l
		}
	}
	if language == "" {
		ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module)
		return false
	}
	node, keyLanguage, err := delivery.ParseToolchainKey(key)
	if err != nil || keyLanguage != language {
		ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err)
		return false
	}
	ix.mu.Lock()
	defer ix.mu.Unlock()
	if deleted {
		if _, had := ix.toolchains[language][node]; !had {
			return false
		}
		delete(ix.toolchains[language], node)
		return true
	}
	tc, err := delivery.DecodeToolchain(value)
	if err != nil || tc.Language != language {
		ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err)
		return false
	}
	if ix.toolchains == nil {
		ix.toolchains = map[string]map[string]delivery.Toolchain{}
	}
	if ix.toolchains[language] == nil {
		ix.toolchains[language] = map[string]delivery.Toolchain{}
	}
	cur, had := ix.toolchains[language][node]
	ix.toolchains[language][node] = tc
	return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
}

// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain
// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing.
func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error {
	if !ix.PutToolchain(module, key, value, deleted) {
		return nil
	}
	return ix.Recompute(ctx)
}

// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
func (ix *Indexer) agreed() map[string]toolchainOrNone {
	ix.mu.Lock()
	defer ix.mu.Unlock()
	out := map[string]toolchainOrNone{}
	for language := range ToolchainModules {
		var entries []delivery.NodeToolchain
		for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) {
			entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]})
		}
		tc, none := delivery.AgreedToolchain(entries)
		out[language] = toolchainOrNone{tc, none}
	}
	return out
}

type toolchainOrNone struct {
	tc   delivery.Toolchain2
	none string
}

// known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at.
type known struct {
	key       string
	cur       delivery.Module
	languages map[string]string
	bases     []string
	contexts  bool
}

// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
// the inputs kept at its merge (Inputs2), its manifest (manifests.<digest>), the fingerprints of its bases and of the
// builder, and the toolchains the build modules report. A module is computed after its bases and the builder, whatever
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
// no kept inputs (a message that carried no manifest) is left as it is.
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil) }

// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is.
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error {
	if ix.Modules == nil {
		return nil
	}
	if ix.Manifests == nil {
		return ix.writeOverlay(ctx, overlay, nil)
	}
	ix.recompute.Lock()
	defer ix.recompute.Unlock()
	keys, err := ix.Modules.Keys(ctx)
	if err != nil {
		return fmt.Errorf("listing the modules: %w", err)
	}
	for key := range overlay {
		if !slices.Contains(keys, key) {
			keys = append(keys, key)
		}
	}
	slices.Sort(keys)
	held := map[string]delivery.Module{}
	all := map[string]*known{}
	for _, key := range keys {
		cur, ok := overlay[key]
		if !ok {
			value, _, found, err := ix.Modules.Get(ctx, key)
			if err != nil {
				return fmt.Errorf("modules.%s: %w", key, err)
			}
			if !found {
				continue
			}
			if cur, err = delivery.DecodeModule(value); err != nil {
				ix.logf("modules.%s holds what is not a module, left: %v", key, err)
				continue
			}
		}
		held[key] = cur
		if cur.Manifest == "" || cur.Inputs2 == nil {
			continue
		}
		m, err := ix.manifestOf(ctx, cur.Manifest)
		if err != nil {
			ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err)
			continue
		}
		all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
	}
	tools := ix.agreed()
	type result struct {
		fp, none string
		tcs      map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
	}
	done := map[string]result{}
	visiting := map[string]bool{}
	var of func(module string, chain []string) result
	of = func(module string, chain []string) result {
		if r, ok := done[module]; ok {
			return r
		}
		k, ours := all[module]
		if !ours { // a module the indexer holds no inputs of: its record, as written
			cur, has := held[module]
			switch {
			case !has:
				return result{fp: "", none: "the indexer holds no record of it"}
			case cur.FP2 != "":
				return result{fp: cur.FP2, none: ""}
			case cur.None2 != "":
				return result{fp: "", none: cur.None2}
			}
			return result{fp: "", none: "its record carries no fingerprint of the new form"}
		}
		if visiting[module] {
			return result{fp: "", none: "a cycle of bases: " + strings.Join(append(chain, module), " -> ")}
		}
		visiting[module] = true
		defer delete(visiting, module)
		chain = append(chain, module)
		kept := *k.cur.Inputs2
		kept.Bases, kept.Builder = nil, ""
		var missing []string
		for _, b := range k.bases {
			if r := of(b, chain); r.fp != "" {
				kept.Bases = append(kept.Bases, r.fp)
			} else {
				missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", b, r.none))
			}
		}
		slices.Sort(kept.Bases)
		slices.Sort(missing)
		toolchains := map[string]delivery.Toolchain2{}
		var noToolchain string
		for _, artifact := range slices.Sorted(maps.Keys(k.languages)) {
			t := tools[k.languages[artifact]]
			if t.none != "" {
				if noToolchain == "" {
					noToolchain = t.none
				}
				continue
			}
			toolchains[artifact] = t.tc
		}
		var noBuilder string
		if module == BuilderModule {
			// Built by build-agent until release R4g: its value of today's form.
			if a := held[BuilderBuiltBy]; a.FP != "" {
				kept.Builder = a.FP
			} else {
				noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None)
			}
		} else if r := of(BuilderModule, chain); r.fp != "" {
			kept.Builder = r.fp
		} else {
			noBuilder = None2Builder
			if _, has := held[BuilderModule]; has {
				noBuilder += ": " + r.none
			}
		}
		var r result
		switch {
		case k.contexts:
			r.none = None2Context
		case len(missing) > 0:
			r.none = strings.Join(missing, "; ")
		case noToolchain != "":
			r.none = noToolchain
		case noBuilder != "":
			r.none = noBuilder
		default:
			r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
			if r.fp != "" {
				r.tcs = toolchainsUsed(toolchains)
			}
		}
		done[module] = r
		k.cur.Inputs2 = &kept
		return r
	}
	finals := map[string]delivery.Module{}
	for _, key := range slices.Sorted(maps.Keys(all)) {
		k := all[key]
		r := of(key, nil)
		if _, merged := overlay[key]; merged {
			final := overlay[key]
			final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs
			// What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge
			// that moves nothing for the module writes nothing (as Recompute does).
			if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
				if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none {
					in := *final.Inputs2
					in.Bases, in.Builder = old.Inputs2.Bases, old.Inputs2.Builder
					final.Inputs2 = &in
				}
			}
			finals[key] = final
			continue
		}
		if k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) {
			continue
		}
		digest, inputs := k.cur.Manifest, k.cur.Inputs2
		err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
			if !found {
				return nil, false, nil
			}
			cur, err := delivery.DecodeModule(value)
			if err != nil || cur.Manifest != digest || cur.Inputs2 == nil {
				return nil, false, nil // a merge changed it meanwhile; its own recompute follows
			}
			if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) {
				return nil, false, nil
			}
			cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs
			cur.Inputs2 = inputs
			data, err := delivery.EncodeModule(cur)
			return data, false, err
		})
		if err != nil {
			return fmt.Errorf("modules.%s: %w", key, err)
		}
	}
	return ix.writeOverlay(ctx, overlay, finals)
}

// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
// the ones that were computed, the others are written as the merge gave them.
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
	for _, key := range slices.Sorted(maps.Keys(overlay)) {
		next := overlay[key]
		if f, ok := finals[key]; ok {
			next = f
		}
		err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
			if found {
				if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
					return nil, false, nil
				}
			}
			data, err := delivery.EncodeModule(next)
			return data, false, err
		})
		if err != nil {
			return fmt.Errorf("modules.%s: %w", key, err)
		}
	}
	return nil
}

// manifestOf is the parsed manifest manifests.<digest> holds.
func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) {
	key, err := delivery.ManifestsKey(digest)
	if err != nil {
		return catalogue.Manifest{}, err
	}
	value, _, found, err := ix.Manifests.Get(ctx, key)
	if err != nil {
		return catalogue.Manifest{}, err
	}
	if !found {
		return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
	}
	kept, err := delivery.DecodeManifest(value)
	if err != nil {
		return catalogue.Manifest{}, err
	}
	return catalogue.ParseManifest([]byte(kept.JSON))
}

// hasContext is whether an artifact of the manifest reads another repository's tree.
func hasContext(m catalogue.Manifest) bool {
	if m.Build == nil {
		return false
	}
	for _, a := range m.Build.Artifacts {
		if a.Context != nil {
			return true
		}
	}
	return false
}

// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
	if len(by) == 0 {
		return nil
	}
	out := map[string]delivery.ToolchainUsed{}
	for _, t := range by {
		out[t.Language] = delivery.ToolchainUsed{Version: t.Version, Recipe: t.Recipe}
	}
	return out
}