mesh-controller/internal/builder/check_builds.go

package builder

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
)

// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
// it did not fail: a change that fails is not merged, and nothing it built would be taken.

// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
type CheckBuildSpec struct {
	Module string
	// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
	// module is built from the repository checked, at the branch merged into.
	Repository string
	Path       string
	Ref        string
	Merged     bool
	Tier       int
	// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
	Skip string
}

// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
type CheckBuilding struct {
	Publish Publisher
	Scratch Scratch
	Held    map[string]string
	// Running is each artifact of the build the mesh has registered for its module, keyed "<module>/<artifact>"
	// (novox/hq issue 721): what a build for the merge is compared with — never Held's newest build, which may be one
	// its gate failed and the mesh put back.
	Running map[string]string
	Npmrc   Npmrc
	Seats   map[string]string
	// Named are the builds the build seat's verdicts of earlier checks of this pull request at this head name
	// (novox/hq issue 703): a build kept under a module's key is kept again, instead of built, only when one of them
	// names its key and every digest, and the build is this pull request's at this head. The record alone is never
	// enough.
	Named []NamedBuild
}

// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
// why it was not built.
type CheckBuilt struct {
	Module string
	Key    string
	Made   []catalogue.Built
	// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
	NotBuilt string
	Took     time.Duration
	// Kept is the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
	// the verdict names that build, by key and digest, as that check's. Empty when this check built it.
	Kept string
	// Same says every artifact came out the same as the build the mesh holds of the module (novox/hq issue 721), and
	// SameAs names, per artifact, the held reference it was compared with: the walk leaves the module where it is
	// while the mesh still holds that build. Empty when it differs, or could not be compared.
	Same   string
	SameAs map[string]string
}

// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
func skipsTheBuilds(v CheckVerdict) string {
	for _, l := range []*Layer{v.Gate, v.Repo} {
		if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
			return "the check did not pass, so nothing is built for its merge"
		}
	}
	return ""
}

// buildForTheMerge builds each module the plan moves into the scratch namespace (makeForTheMerge), and keeps what it
// built under its keys (keepForTheMerge): the order of a check that builds after it has judged.
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
	judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
	out, kept := makeForTheMerge(ctx, run, spec, with, workspace, tree, merge, judged, forge, log, say)
	return keepForTheMerge(ctx, spec, with, out, kept, say)
}

// makeForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace — **the modules of a
// tier at the same time** (novox/hq issue 716): none of them stands on another, and a tier waits only for the ones
// before it, whose builds it is handed. A module standing after one not built is not built either: what it stands on
// in the walk is not what it would stand on here. tree is the check's copy of the merge and merge the merge commit it
// made there; workspace is where each build's working copy is made, one directory per module, which the host's
// container engine sees at the same path (novox/hq issue 713). Answers what each module came to, in the plan's order,
// and the builds to keep — nothing is kept here.
func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
	judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) ([]CheckBuilt, []CheckBuild) {
	if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
		return nil, nil
	}
	// What an earlier check left, ended before it could remove it, is not built on.
	place := filepath.Join(workspace, "for-the-merge")
	_ = os.RemoveAll(place)
	defer os.RemoveAll(place)
	// One log, said to by builds running at once: a line at a time.
	var saying sync.Mutex
	if log != nil {
		inner := log
		log = func(step, message string) {
			saying.Lock()
			defer saying.Unlock()
			inner(step, message)
		}
	}
	inner := say
	say = func(step, format string, args ...any) {
		saying.Lock()
		defer saying.Unlock()
		inner(step, format, args...)
	}
	ref := merge
	if ref == "" {
		ref = spec.Ref
	}
	held := map[string]string{}
	for k, v := range with.Held {
		held[k] = v
	}
	out := make([]CheckBuilt, len(spec.Builds))
	results := make([]Result, len(spec.Builds))
	var kept []CheckBuild
	stopAt, stopWhy := -1, ""
	for start := 0; start < len(spec.Builds); {
		tier := spec.Builds[start].Tier
		end := start
		for end < len(spec.Builds) && spec.Builds[end].Tier == tier {
			end++
		}
		// What this tier's builds are handed: the builds of the tiers before it, read alone while they run.
		handed := map[string]string{}
		for k, v := range held {
			handed[k] = v
		}
		var building sync.WaitGroup
		for i := start; i < end; i++ {
			b := spec.Builds[i]
			if stopAt >= 0 && b.Tier > stopAt {
				out[i] = CheckBuilt{Module: b.Module, NotBuilt: stopWhy}
				continue
			}
			if b.Skip != "" {
				say("build", "%s not built for the merge: %s", b.Module, b.Skip)
				out[i] = CheckBuilt{Module: b.Module, NotBuilt: b.Skip}
				continue
			}
			repository, at := b.Repository, b.Ref
			if b.Merged {
				repository, at = tree, ref
			}
			name := b.Module
			if !safeName.MatchString(name) {
				name = fmt.Sprintf("module-%d", i)
			}
			building.Add(1)
			go func(i int, b CheckBuildSpec, repository, at, dir string) {
				defer building.Done()
				began := time.Now()
				say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
				var mirrored []string
				result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at, dir, handed,
					with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true, Scratch: with.Scratch, Named: with.Named,
						pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, head: judged.Head},
					with.Seats)
				built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
				switch {
				case errors.Is(err, errNoKey):
					built.NotBuilt = err.Error()
				case err != nil:
					built.NotBuilt = "its build failed: " + err.Error()
				case result.ReuseKey == "":
					built.NotBuilt = "it has no reuse key: " + result.NoKey
				}
				if built.NotBuilt == "" {
					// **Compared with what runs** (novox/hq issue 721): a failed comparison says nothing, and the
					// module moves as before.
					same, against, err := sameAsHeld(ctx, run, dir, b.Module, result.Built, with.Running)
					switch {
					case err != nil:
						say("build", "%s could not be compared with the build the mesh holds, so it moves: %v", b.Module, err)
					case same != "":
						built.Same, built.SameAs = same, against
						say("build", "%s %s", b.Module, same)
					}
				}
				out[i], results[i] = built, result
			}(i, b, repository, at, filepath.Join(place, name))
		}
		building.Wait()
		for i := start; i < end; i++ {
			b, built, result := spec.Builds[i], &out[i], results[i]
			if built.NotBuilt != "" {
				if built.Key != "" || built.Took > 0 {
					say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
				}
				if stopAt < 0 || b.Tier < stopAt {
					stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
				}
				continue
			}
			built.Made = result.Built
			for _, m := range result.Built {
				held[b.Module+"/"+m.Name] = m.Reference
			}
			if result.Reused != nil {
				// **Kept, not built** (novox/hq issue 703): the earlier check's build, named as that check's, so the
				// walk takes it as the verdict names it; this check's document names it too, so it stays while this
				// check does (ADR 0350 §5).
				built.Kept = result.Reused.From.Check
				say("build", "%s kept for the merge under %s: %s", b.Module, twelve(result.ReuseKey[3:]),
					result.Reused.From.Said())
				kept = append(kept, result.Reused.From)
				continue
			}
			say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]),
				built.Took.Round(time.Second))
			if len(result.Built) > 0 {
				kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
					Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
					Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
			}
		}
		start = end
	}
	return out, kept
}

// keepForTheMerge keeps a check's builds under their keys, its own document naming them, so the walk of its merge
// finds them; what cannot be kept is let go of, said, and each module says it — never the check's verdict.
func keepForTheMerge(ctx context.Context, spec CheckSpec, with CheckBuilding, out []CheckBuilt, kept []CheckBuild,
	say func(step, format string, args ...any)) []CheckBuilt {
	if len(kept) == 0 {
		return out
	}
	if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
		time.Now().UTC(), kept); err != nil {
		// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
		say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
		if err := with.Scratch.Discard(ctx, kept); err != nil {
			say("build", "and what they made could not all be let go of: %v", err)
		}
		for i := range out {
			if out[i].NotBuilt == "" {
				out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
			}
		}
	}
	return out
}

// discardForTheMerge lets go of builds a check made beside its tests when the check did not pass (novox/hq issue 716):
// a change that fails is not merged, and nothing it built is kept. A build another check names stays (Discard).
func discardForTheMerge(ctx context.Context, with CheckBuilding, kept []CheckBuild, why string,
	say func(step, format string, args ...any)) {
	say("check", "%s: the %d build(s) made beside it are let go of", why, len(kept))
	if len(kept) == 0 {
		return
	}
	if err := with.Scratch.Discard(ctx, kept); err != nil {
		say("build", "what they made could not all be let go of, and goes with the week's sweep: %v", err)
	}
}

// besideTheTests are a check's builds for its merge running while its gate and its own check run (novox/hq issue
// 716): started once every clone is made, waited for when the check has judged, kept when it passed and let go of
// when it did not.
type besideTheTests struct {
	spec    CheckSpec
	with    CheckBuilding
	cancel  context.CancelFunc
	done    chan struct{}
	place   string
	out     []CheckBuilt
	kept    []CheckBuild
	began   time.Time
	settled bool
	say     func(step, format string, args ...any)
}

// buildsBesideTheTests is where a check's builds run beside its tests: a directory made for this check under the judge
// cache's homes/ (ADR 0358 §6), on the host path the build agent declares — so the host's container engine mounts what
// the agent sees (issue 713) — and never mounted by a check's container, so nothing a pull request's script runs can
// change a build while it is made (issue 703). Empty, and the builds wait for the check, without a judge cache.
func buildsBesideTheTests(cache, workspace, id string) (string, error) {
	if cache == "" {
		return "", nil
	}
	if withinDir(workspace, cache) {
		return "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
			"nothing is built for the merge where a pull request could reach it", cache, workspace)
	}
	if !safeName.MatchString(id) {
		return "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
	}
	place := filepath.Join(cache, "homes", id+"-builds")
	if err := os.RemoveAll(place); err != nil {
		return "", err
	}
	if err := os.MkdirAll(place, 0o700); err != nil {
		return "", err
	}
	return place, nil
}

// startBesideTheTests starts the builds for the merge in place, from merged, the check's copy of the merge.
func startBesideTheTests(ctx context.Context, run Runner, spec CheckSpec, place, merged string, judged CheckVerdict,
	forge GitCredential, log Log, say func(step, format string, args ...any)) *besideTheTests {
	building, cancel := context.WithCancel(ctx)
	b := &besideTheTests{spec: spec, with: *spec.BuildWith, cancel: cancel, done: make(chan struct{}), place: place,
		began: time.Now(), say: say}
	say("check", "building what the merge moves (%d module(s)) beside its tests", len(spec.Builds))
	go func() {
		defer close(b.done)
		b.out, b.kept = makeForTheMerge(building, run, spec, b.with, place, merged, judged.Merge, judged, forge, log, say)
	}()
	return b
}

// settle waits for the builds and keeps them when the check passed; a check that did not pass lets go of them, and
// its verdict names none.
func (b *besideTheTests) settle(ctx context.Context, verdict CheckVerdict) []CheckBuilt {
	<-b.done
	b.settled = true
	defer os.RemoveAll(b.place)
	if why := skipsTheBuilds(verdict); why != "" {
		discardForTheMerge(ctx, b.with, b.kept, why, b.say)
		return nil
	}
	built := keepForTheMerge(ctx, b.spec, b.with, b.out, b.kept, b.say)
	b.say("check", "built for the merge: %d of %d module(s), beside the tests (%s)", builtCount(built), len(built),
		time.Since(b.began).Round(time.Second))
	return built
}

// abandon ends builds a check that ended without a verdict leaves, and lets go of what they made.
func (b *besideTheTests) abandon() {
	if b.settled {
		return
	}
	b.cancel()
	<-b.done
	defer os.RemoveAll(b.place)
	letGo, done := context.WithTimeout(context.Background(), time.Minute)
	defer done()
	discardForTheMerge(letGo, b.with, b.kept, "the check ended without a verdict", b.say)
}