mesh-delivery/modules/mesh-delivery-builder/internal/build/run.go

package build

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// ToolchainEntries are the nodes' toolchain entries of a language (ToolchainSet).
type ToolchainEntries interface {
	Entries(language string) []delivery.NodeToolchain
}

// toolPrefix is the start of each build module's tool names: <prefix>fetch and <prefix>build.
var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule: "build_node_"}

// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
// artifact to build claims the fingerprint's build key, builds it through the build modules on the nodes
// the toolchains choose, and writes the key done or failed. It holds no timer; a build starts from a modules value or a
// change of the toolchains.
type Builder struct {
	Claimer    Claimer
	Runner     Runner
	Toolchains ToolchainEntries
	// Presence says which nodes are online (the watchman's); a node that is not is not chosen.
	Presence  interface{ Online(node string) bool }
	Manifests Manifests
	// Health says whether a node's build module is running and healthy (the node-engine's have); a node whose build module
	// is not is not chosen. Nil means every node's is.
	Health interface {
		Healthy(node, module string) bool
	}
	// Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
	Places *Places
	// Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4).
	Nodes int
	// Limit is how many builds run at once; four when 0.
	Limit int
	Logf  func(format string, args ...any)

	once     sync.Once
	sem      chan struct{}
	wg       sync.WaitGroup
	mu       sync.Mutex
	known    map[string]delivery.Module // by modules key
	inflight map[string]bool            // by build key
	again    map[string]delivery.Module // by build key: the newest value that arrived while the key was being built
	plans    map[string]planned         // by manifest digest
	tried    map[string]bool            // by build key: looked at since this process started
}

type planned struct {
	plan Plan
	err  error
}

func (b *Builder) init() {
	b.once.Do(func() {
		limit := b.Limit
		if limit <= 0 {
			limit = 4
		}
		b.sem = make(chan struct{}, limit)
		b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
		if b.Places == nil {
			b.Places = &Places{}
		}
	})
}

func (b *Builder) logf(format string, args ...any) {
	if b.Logf != nil {
		b.Logf(format, args...)
	}
}

// Wait returns when every build started so far has ended.
func (b *Builder) Wait() { b.wg.Wait() }

// Module takes one value of the indexer's modules bucket in (the key is the module's name) and builds it if it is
// to be built. A value that is not a module is said and left.
func (b *Builder) Module(ctx context.Context, key string, value []byte) {
	b.init()
	m, err := delivery.DecodeModule(value)
	if err != nil {
		b.logf("modules.%s holds what is not a module, left: %v", key, err)
		return
	}
	b.mu.Lock()
	b.known[key] = m
	b.mu.Unlock()
	b.consider(ctx, key, m)
}

// Forget takes a module out (its key was deleted).
func (b *Builder) Forget(key string) {
	b.init()
	b.mu.Lock()
	delete(b.known, key)
	b.mu.Unlock()
}

// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
func (b *Builder) Reconsider(ctx context.Context) {
	b.init()
	b.mu.Lock()
	known := make(map[string]delivery.Module, len(b.known))
	for k, m := range b.known {
		known[k] = m
	}
	b.mu.Unlock()
	for _, k := range sortedKeys(known) {
		b.consider(ctx, k, known[k])
	}
}

func sortedKeys(m map[string]delivery.Module) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// planOf is the plan of the manifest with this digest, read once: a manifest by digest never changes. A manifest that
// cannot be read is not kept, so it is read again when the module is.
func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool) {
	b.mu.Lock()
	p, ok := b.plans[digest]
	b.mu.Unlock()
	if ok {
		return p.plan, p.err, true
	}
	m, err := b.Manifests.Manifest(ctx, digest)
	if err != nil {
		return Plan{}, fmt.Errorf("its manifest %s cannot be read: %w", digest, err), false
	}
	plan, err := PlanOf(m)
	b.mu.Lock()
	b.plans[digest] = planned{plan, err}
	b.mu.Unlock()
	return plan, err, true
}

func (b *Builder) consider(ctx context.Context, module string, m delivery.Module) {
	if m.FP2 == "" || m.Manifest == "" {
		return
	}
	key, err := delivery.BuilderBuildKey(module, m.FP2)
	if err != nil {
		b.logf("modules.%s: %v", module, err)
		return
	}
	b.mu.Lock()
	if b.inflight[key] {
		b.again[key] = m // looked at once more when the build ends, so a change is never lost
		b.mu.Unlock()
		return
	}
	b.inflight[key] = true
	b.mu.Unlock()
	b.wg.Add(1)
	go func() {
		defer b.wg.Done()
		b.sem <- struct{}{}
		defer func() { <-b.sem }()
		for {
			b.try(ctx, module, m)
			b.mu.Lock()
			next, more := b.again[key]
			delete(b.again, key)
			if !more || ctx.Err() != nil {
				delete(b.inflight, key)
				b.mu.Unlock()
				return
			}
			b.mu.Unlock()
			m = next
		}
	}()
}

// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
	plan, planErr, read := b.planOf(ctx, m.Manifest)
	if read && planErr == nil && len(plan.Tasks) == 0 {
		return // nothing in the manifest for a builder to build
	}
	if planErr == nil {
		if why := b.stale(m, plan); why != "" {
			// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
			// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
			b.logf("%s at %s is not built: %s", module, m.FP2, why)
			return
		}
	}
	key, _ := delivery.BuilderBuildKey(module, m.FP2)
	b.mu.Lock()
	first := !b.tried[key]
	b.tried[key] = true
	b.mu.Unlock()
	cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: func(modules []string) bool { return b.moreNodes(modules) }})
	if err != nil {
		b.logf("%s: %v", module, err)
		return
	}
	if cl == nil {
		return
	}
	result := b.build(ctx, module, m, plan, planErr)
	if ctx.Err() != nil {
		return
	}
	if result.State == "" {
		b.logf("%s: left building, the toolchains moved under it", cl.Key)
		return
	}
	result.By, result.Manifest = b.Claimer.By, m.Manifest
	if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, DigestsDifferPrefix) {
		// A done key built again on a second node keeps its done value when the new build fails for another reason than
		// the digests: the first build stands, and a later change of the nodes looks again.
		b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error)
		return
	}
	if err := b.Claimer.Finish(ctx, cl, result); err != nil {
		b.logf("%s: the build could not be written: %v", cl.Key, err)
		return
	}
	if result.State == delivery.BuildFailed {
		b.logf("%s: failed: %s", cl.Key, result.Error)
	}
}

// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
func (b *Builder) stale(m delivery.Module, plan Plan) string {
	for _, bm := range plan.Modules() {
		if !plan.NeedsToolchain(bm) {
			continue // an archive is packed with no toolchain
		}
		language := LanguageOf[bm]
		tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
		if none != "" {
			return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
		}
		want, ok := m.Toolchains[language]
		if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
			return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
		}
	}
	return ""
}

func failed(format string, args ...any) delivery.BuilderBuild {
	return delivery.BuilderBuild{State: delivery.BuildFailed, Error: fmt.Sprintf(format, args...)}
}

// lane is the build of a plan's tasks of one build module on one node.
type lane struct {
	module, node string
	tasks        []Task
	digests      map[string]string // artifact -> digest
	err          error
}

func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error) delivery.BuilderBuild {
	switch {
	case planErr != nil:
		return failed("%s", planErr)
	case m.Path == "":
		// Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615).
		return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module)
	}
	repository := delivery.RepositoryName(m.Repository)
	var lanes []*lane
	used := map[string]delivery.ToolchainUsed{}
	var steps, oneNode []string
	// Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up.
	giveUp := func(nodes ...string) {
		for _, n := range nodes {
			b.Places.Cancel(n)
		}
	}
	giveUpLanes := func() {
		for _, l := range lanes {
			b.Places.Cancel(l.node)
		}
	}
	for _, bm := range plan.Modules() {
		language := LanguageOf[bm]
		entries := b.Toolchains.Entries(language)
		qualifies := func(node string) bool { return b.qualifies(node, bm) }
		var nodes []string
		if plan.NeedsToolchain(bm) {
			ns, tc, err := ChooseNodes(language, entries, b.Places, qualifies, b.Nodes)
			if err != nil {
				giveUpLanes()
				return failed("%s", err)
			}
			if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
				giveUp(ns...)
				giveUpLanes()
				return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
			}
			used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
			nodes = ns
		} else {
			// An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies.
			ns, err := ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes)
			if err != nil {
				giveUpLanes()
				return failed("%s", err)
			}
			nodes = ns
		}
		if len(nodes) == 1 {
			oneNode = append(oneNode, bm)
			steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0]))
		} else {
			sorted := append([]string(nil), nodes...)
			sort.Strings(sorted)
			steps = append(steps, fmt.Sprintf("%s on %s", bm, strings.Join(sorted, " and ")))
		}
		var tasks []Task
		for _, t := range plan.Tasks {
			if t.Module == bm {
				tasks = append(tasks, t)
			}
		}
		for _, n := range nodes {
			lanes = append(lanes, &lane{module: bm, node: n, tasks: tasks, digests: map[string]string{}})
		}
	}
	var wg sync.WaitGroup
	for _, l := range lanes {
		wg.Add(1)
		go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module) }()
	}
	wg.Wait()
	var errs []string
	for _, l := range lanes {
		if l.err != nil {
			errs = append(errs, l.err.Error())
		}
	}
	if len(errs) > 0 {
		return failed("%s", strings.Join(errs, "; "))
	}
	var artifacts []delivery.BuiltArtifact
	for _, t := range plan.Tasks {
		byNode := map[string]string{}
		for _, l := range lanes {
			if d, ok := l.digests[t.Artifact]; ok && l.module == t.Module {
				byNode[l.node] = d
			}
		}
		d, err := Agree(t.Artifact, byNode)
		if err != nil {
			return failed("%s", err)
		}
		artifacts = append(artifacts, delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d})
	}
	step := "built " + strings.Join(steps, "; ")
	if len(oneNode) > 0 {
		// Marked so that Claim builds the key again once a second node qualifies (ADR 0478 clause 4).
		step = OneNodePrefix + strings.Join(oneNode, ", ") + "): " + step
	}
	if len(used) == 0 {
		used = nil
	}
	return delivery.BuilderBuild{State: delivery.BuildDone, Step: step, Artifacts: artifacts, Toolchains: used}
}

func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) }

// qualifies is whether the node can build with the build module now: it is online and the build module on it is healthy.
func (b *Builder) qualifies(node, bm string) bool {
	return b.online(node) && (b.Health == nil || b.Health.Healthy(node, bm))
}

// moreNodes is whether any of these build modules has two nodes that qualify now.
func (b *Builder) moreNodes(modules []string) bool {
	for _, bm := range modules {
		seen := map[string]bool{}
		n := 0
		for _, e := range b.Toolchains.Entries(LanguageOf[bm]) {
			if !seen[e.Node] && b.qualifies(e.Node, bm) {
				seen[e.Node] = true
				n++
			}
		}
		if n >= 2 {
			return true
		}
	}
	return false
}

// step runs one step of a build module; the lane that runs it holds its node's place.
func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) {
	return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args)
}

// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time.
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) {
	release, err := b.Places.Acquire(ctx, l.node)
	if err != nil {
		l.err = err
		return
	}
	defer release()
	raw, err := b.step(ctx, l.module, l.node, "fetch", map[string]any{"repository": repository, "commit": m.Commit})
	if err != nil {
		l.err = err
		return
	}
	var fetched struct {
		Workspace string `json:"workspace"`
	}
	if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
		l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
		return
	}
	for _, t := range l.tasks {
		raw, err := b.step(ctx, l.module, l.node, "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact})
		if err != nil {
			l.err = err
			return
		}
		var built struct {
			Digest string `json:"digest"`
		}
		if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
			l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw)
			return
		}
		l.digests[t.Artifact] = built.Digest
	}
}