Build a change once: the check builds what its merge moves, the walk takes it by key

A merge built every moved module a second time although its check had judged
the same tree (issue 649). The check now builds into the scratch namespace
under a reuse key (source fingerprint plus the builder's own digest), and the
walk of the trunk commit copies a build of the same key by digest instead of
building, saying per module why when it cannot (novox/hq ADR 0350).

internal/builder/reuse.go

@@ -0,0 +1,539 @@
+package builder
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// One build per change (novox/hq ADR 0350).
+//
+// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
+// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
+// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
+// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
+// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
+// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
+// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
+// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
+//
+// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
+// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
+// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
+// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
+// key, and is always built.
+
+// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
+// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
+// key, every build its own — when it cannot be said.
+var Version string
+
+// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
+// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
+// builder would read as one).
+func SelfVersion(stamped string) string {
+	if exe, err := os.Executable(); err == nil {
+		if f, err := os.Open(exe); err == nil {
+			defer f.Close()
+			sum := sha256.New()
+			if _, err := io.Copy(sum, f); err == nil {
+				return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
+			}
+		}
+	}
+	if stamped == "" || stamped == "development" {
+		return ""
+	}
+	return stamped
+}
+
+// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
+// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
+const ScratchPrefix = "scratch/"
+
+// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
+// What the holder lets go of by (§5).
+const checksRepository = ScratchPrefix + "checks"
+
+// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
+const reuseTagPrefix = "r1-"
+
+// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
+const CheckBuildsKept = 7 * 24 * time.Hour
+
+// The parts of a reuse key, as the walk names the one that differs (§3).
+const (
+	partModule   = "the module"
+	partTree     = "the build source"
+	partBases    = "a base image"
+	partBuilder  = "the builder's version"
+	partContext  = "a context"
+	partTool     = "a toolchain"
+	partUpstream = "an upstream image"
+)
+
+// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
+type reuseKey struct {
+	Key   string
+	Parts map[string]string
+	Why   string
+}
+
+// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
+func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
+	if src == nil || src.unpinned != "" || src.tree == "" {
+		why := "its source does not pin the build"
+		if src != nil && src.unpinned != "" {
+			why = src.unpinned
+		} else if src != nil && src.tree == "" {
+			why = "its tree could not be named"
+		}
+		return reuseKey{Why: why}
+	}
+	if version == "" {
+		return reuseKey{Why: "the builder's version cannot be said"}
+	}
+	parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
+	var bases []string
+	seen := map[string]bool{}
+	for _, b := range src.bases {
+		if d := referenceDigest(b); !seen[d] {
+			seen[d] = true
+			bases = append(bases, d)
+		}
+	}
+	sort.Strings(bases)
+	parts[partBases] = strings.Join(bases, " ")
+	for a, t := range src.contexts {
+		parts[partContext+" "+a] = t
+	}
+	for a, t := range src.toolchains {
+		parts[partTool+" "+a] = t
+	}
+	// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
+	// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
+	if manifest.Build != nil {
+		for _, a := range manifest.Build.Artifacts {
+			if a.Kind != catalogue.ArtifactUpstream {
+				continue
+			}
+			if !strings.Contains(a.From, "@sha256:") {
+				return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
+			}
+			parts[partUpstream+" "+a.Name] = a.From
+		}
+	}
+	lines := make([]string, 0, len(parts))
+	for k, v := range parts {
+		lines = append(lines, k+"="+v)
+	}
+	sort.Strings(lines)
+	sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
+	return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
+}
+
+// tagOf is the tag a key's record is kept under.
+func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
+
+// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
+func differing(was, now map[string]string) []string {
+	names := map[string]bool{}
+	for k := range was {
+		names[k] = true
+	}
+	for k := range now {
+		names[k] = true
+	}
+	var out []string
+	for k := range names {
+		if was[k] != now[k] {
+			out = append(out, k)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// PullRef names a pull request.
+type PullRef struct {
+	Owner  string `json:"owner"`
+	Repo   string `json:"repo"`
+	Number int    `json:"number"`
+}
+
+func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
+
+// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
+func (p PullRef) slug() string {
+	clean := func(s string) string {
+		return strings.Map(func(r rune) rune {
+			if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
+				return r
+			}
+			return '-'
+		}, s)
+	}
+	return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
+}
+
+// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
+type CheckBuild struct {
+	Key    string            `json:"key"`
+	Module string            `json:"module"`
+	Parts  map[string]string `json:"parts"`
+	// Made are its artifacts, in the scratch namespace.
+	Made []catalogue.Built `json:"made"`
+	// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
+	// check's ask; At when it was built.
+	Pull  PullRef   `json:"pull"`
+	Head  string    `json:"head,omitempty"`
+	Trunk string    `json:"trunk,omitempty"`
+	Check string    `json:"check"`
+	At    time.Time `json:"at"`
+}
+
+// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
+func (c CheckBuild) Said() string {
+	s := "the check of " + c.Pull.String()
+	if c.Head != "" {
+		s += " at head " + twelve(c.Head)
+	}
+	if c.Trunk != "" {
+		s += " (merged into trunk " + twelve(c.Trunk) + ")"
+	}
+	return s
+}
+
+// checkIndex is one check's document under checksRepository: every build it made.
+type checkIndex struct {
+	Pull   PullRef      `json:"pull"`
+	Check  string       `json:"check"`
+	At     time.Time    `json:"at"`
+	Builds []CheckBuild `json:"builds"`
+}
+
+// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
+// Registry is one; a test's is a map.
+type Scratch interface {
+	// Keep records a check's builds: each under its module and key, and the check's own document.
+	Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
+	// Find is the check build kept under a module's key; false when none is.
+	Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
+	// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
+	// key from.
+	OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
+	// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
+	Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
+	// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
+	// each answers how many checks it let go of.
+	Forget(ctx context.Context, pull PullRef) (int, error)
+	Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
+	// Discard lets go of what builds a check could not keep made: nothing would ever find them.
+	Discard(ctx context.Context, builds []CheckBuild) error
+}
+
+// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
+// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
+type scratchPublisher struct{ to Publisher }
+
+func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
+	return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
+}
+
+func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
+	return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
+}
+
+func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
+	m, can := s.to.(Mirrorer)
+	if !can {
+		return "", errors.New("the artifact store here cannot copy an upstream image")
+	}
+	return m.MirrorImage(ctx, from, ScratchPrefix+repository)
+}
+
+func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
+	if m, can := s.to.(BaseMirrorer); can {
+		return m.MirrorBase(ctx, from, repository, former)
+	}
+	if m, can := s.to.(Mirrorer); can {
+		return m.MirrorImage(ctx, from, repository)
+	}
+	return "", errors.New("the artifact store here cannot copy a base")
+}
+
+// The registry as the scratch namespace.
+
+func (r Registry) store() artifacts.Store {
+	return artifacts.Store{Address: r.Address, HTTP: r.client()}
+}
+
+const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
+
+// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
+func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
+	store := r.store()
+	for _, b := range builds {
+		body, err := json.Marshal(b)
+		if err != nil {
+			return err
+		}
+		if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
+			return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
+		}
+	}
+	body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
+	if err != nil {
+		return err
+	}
+	if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
+		return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
+	}
+	return nil
+}
+
+// Find reads the record kept under a module's key.
+func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
+	body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
+	if errors.Is(err, artifacts.ErrNoTag) {
+		return CheckBuild{}, false, nil
+	}
+	if err != nil {
+		return CheckBuild{}, false, err
+	}
+	var b CheckBuild
+	if err := json.Unmarshal(body, &b); err != nil {
+		return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
+	}
+	if b.Key != key || b.Module != module {
+		return CheckBuild{}, false, nil
+	}
+	return b, true, nil
+}
+
+// checks are the check documents kept, by tag; only those of the pull requests named when any are.
+func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
+	store := r.store()
+	tags, err := store.Tags(ctx, checksRepository)
+	if err != nil {
+		return nil, err
+	}
+	wanted := map[string]PullRef{}
+	for _, p := range pulls {
+		wanted[p.slug()+"."] = p
+	}
+	out := map[string]checkIndex{}
+	for _, tag := range tags {
+		if len(pulls) > 0 {
+			match := false
+			for prefix := range wanted {
+				match = match || strings.HasPrefix(tag, prefix)
+			}
+			if !match {
+				continue
+			}
+		}
+		body, _, err := store.GetTagged(ctx, checksRepository, tag)
+		if errors.Is(err, artifacts.ErrNoTag) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		var c checkIndex
+		if err := json.Unmarshal(body, &c); err != nil {
+			continue // not a document this wrote: left to nobody, said by nothing — it names no build
+		}
+		if len(pulls) > 0 {
+			if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
+				continue // a slug two pull requests share: only the one named
+			}
+		}
+		out[tag] = c
+	}
+	return out, nil
+}
+
+// OfPulls are the check builds of those pull requests, newest first.
+func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
+	if len(pulls) == 0 {
+		return nil, nil
+	}
+	checks, err := r.checks(ctx, pulls)
+	if err != nil {
+		return nil, err
+	}
+	var out []CheckBuild
+	for _, c := range checks {
+		out = append(out, c.Builds...)
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
+	return out, nil
+}
+
+// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
+// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
+func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
+	repository := module + "/" + made.Name
+	from := ScratchPrefix + repository
+	if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
+		digest := made.Digest
+		if digest == "" {
+			_, digest, _ = strings.Cut(made.Reference, "/blobs/")
+		}
+		where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
+		if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
+			return catalogue.Built{}, err
+		}
+		size, err := r.blobSize(ctx, repository, digest)
+		if err != nil {
+			return catalogue.Built{}, err
+		}
+		if err := r.hold(ctx, repository, digest, int(size)); err != nil {
+			return catalogue.Built{}, err
+		}
+		return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
+			Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
+	}
+	_, digest, pinned := strings.Cut(made.Reference, "@")
+	if !pinned || !strings.HasPrefix(digest, "sha256:") {
+		return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
+	}
+	reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
+	if err != nil {
+		return catalogue.Built{}, err
+	}
+	if !strings.HasSuffix(reference, "@"+digest) {
+		return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
+	}
+	return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
+}
+
+// blobSize is the size the store says a blob has.
+func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
+	if err != nil {
+		return 0, err
+	}
+	response, err := r.client().Do(request)
+	if err != nil {
+		return 0, err
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
+		return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
+	}
+	return response.ContentLength, nil
+}
+
+// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
+// record under its key when it is still that check's, and the check's document.
+func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
+	checks, err := r.checks(ctx, []PullRef{pull})
+	if err != nil {
+		return 0, err
+	}
+	return r.letGo(ctx, checks)
+}
+
+// Expire lets go of every check older than the bound (§5).
+func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
+	checks, err := r.checks(ctx, nil)
+	if err != nil {
+		return 0, err
+	}
+	old := map[string]checkIndex{}
+	for tag, c := range checks {
+		if now.Sub(c.At) > older {
+			old[tag] = c
+		}
+	}
+	return r.letGo(ctx, old)
+}
+
+// Discard lets go of builds no check document names.
+func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
+	_, err := r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}})
+	return err
+}
+
+// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
+// its document, so the next sweep asks again.
+func (r Registry) letGo(ctx context.Context, checks map[string]checkIndex) (int, error) {
+	store := r.store()
+	gone := 0
+	var failed []string
+	for tag, c := range checks {
+		ok := true
+		for _, b := range c.Builds {
+			for _, m := range b.Made {
+				reference := catalogue.Recorded(m.Reference)
+				if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
+					continue // never anything outside the scratch namespace
+				}
+				if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
+					ok = false
+					failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
+				}
+			}
+			// The record under the key, only while it is still this check's: a later check of the same key
+			// replaced it, and that one is its own check's to let go of.
+			if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
+				if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
+					ok = false
+					failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
+				}
+			}
+		}
+		if !ok {
+			continue
+		}
+		if tag == "" {
+			gone++
+			continue
+		}
+		if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
+			failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
+			continue
+		}
+		gone++
+	}
+	if len(failed) > 0 {
+		sort.Strings(failed)
+		return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
+	}
+	return gone, nil
+}
+
+// Reused is what the walk took from a check instead of building (§2, §4).
+type Reused struct {
+	From CheckBuild
+	// Made are the copies in the module's namespace: what is registered.
+	Made []catalogue.Built
+}
+
+// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
+func (r Reused) Words() string {
+	var digests []string
+	for _, m := range r.Made {
+		d := m.Digest
+		if d == "" {
+			_, d, _ = strings.Cut(m.Reference, "@")
+		}
+		digests = append(digests, m.Name+" "+d)
+	}
+	return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
+}