mesh-controller/internal/inventory/builds.go

package inventory

import (
	"context"
	"encoding/json"
	"errors"
	"sort"
	"time"

	"github.com/jackc/pgx/v5"
)

// What has been built.
//
// A build result was answered to whoever asked and kept nowhere, so "when did this last build",
// "why did it fail" and "which machine built what is running" had no answer. Failures are recorded
// too: one that leaves no trace is indistinguishable from a build nobody asked for, and the
// difference is the whole of whether somebody should be looking at something.

// Build is one attempt, whichever way it went.
type Build struct {
	ID         string
	Repository string
	Ref        string
	// Module is empty for a build that failed before it knew what it was building.
	Module string
	Commit string
	// On is the machine that did it.
	On string
	// Path is where inside the repository the module lives (novox/hq ADR 0069).
	Path string
	// Manifest is the declaration the builder resolved, as it announced it.
	//
	// **Kept because the catalogue may not have been listening.** The announcement carries this
	// and the catalogue turns it into the module's requires/provides edges. On a fresh mesh the
	// modules built before the catalogue exists are exactly the ones it most needs, so the mesh
	// has to be able to say afterwards what they declared (novox/hq 04-ISSUES/050).
	Manifest []byte
	// Against is every artifact this build stood on, as references rather than module names —
	// what makes a build edge derived rather than declared (ADR 0009).
	Against []string
	// Mirrored is every base this build copied into the artifact store (novox/hq ADR 0257), said
	// whether the build worked or not: the copy is the mesh's, recorded in artifact_mirrored.
	Mirrored []string
	// Read is every repository this build read source from besides the module's own (novox/hq
	// 04-ISSUES/131), at the ref it read.
	Read []ReadRepository
	// SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq
	// issue 280); empty from a builder that predates it, or where the source does not pin the build.
	SourceFingerprint string
	// Failed is the builder's own words, empty when it worked.
	Failed string
	Made   []Artifact
	// Asked is when the build was requested, zero when that is not known (an id of another shape,
	// or a build recorded before the mesh kept it). **What orders one build of a module against
	// another** (novox/hq 04-ISSUES/219): builds in flight together finish in any order, and the
	// one asked last stood on the newest bases.
	Asked time.Time
	// At is when the outcome was recorded — when it finished, not when it was asked.
	At time.Time
}

// AskedOrAt is when the build was asked, or when it was recorded when that is not known — the
// order the mesh had before it kept the request time.
func (b Build) AskedOrAt() time.Time {
	if !b.Asked.IsZero() {
		return b.Asked
	}
	return b.At
}

// newestRequestFirst is the ordering every "what a module currently is" question uses: the newest
// request wins, whenever it finished (novox/hq 04-ISSUES/219). A build whose request time is not
// known is placed at the moment it was recorded, which is the rule that held before.
const newestRequestFirst = `coalesce(asked, at) desc, at desc`

// ReadRepository is a repository a build read source from besides the module's own.
type ReadRepository struct {
	Repository string `json:"repository"`
	Ref        string `json:"ref,omitempty"`
}

// Artifact is one thing a build published.
type Artifact struct {
	Name      string `json:"name"`
	Kind      string `json:"kind"`
	Reference string `json:"reference"`
}

// Worked reports whether this build produced something.
func (b Build) Worked() bool { return b.Failed == "" }

// RecordBuild keeps what a builder said.
//
// Idempotent on the correlation id, because a result can arrive twice: once as the answer to
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
// two, and which of the two is real is not a question anybody could answer afterwards.
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
	made, err := json.Marshal(b.Made)
	if err != nil {
		return err
	}
	against, err := json.Marshal(b.Against)
	if err != nil {
		return err
	}
	read, err := json.Marshal(b.Read)
	if err != nil {
		return err
	}
	var module *string
	if b.Module != "" {
		module = &b.Module
	}
	var asked *time.Time
	if !b.Asked.IsZero() {
		asked = &b.Asked
	}
	_, err = i.store.Pool().Exec(ctx,
		`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
		                    source_path, manifest, built_against, built_contexts, asked, source_fingerprint)
		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
		 on conflict (id) do nothing`,
		b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
		b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint)
	if err != nil {
		return err
	}
	return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
}

// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
// by the build that said so, or by a person who says why. The first record of a copy stands.
func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, references []string) error {
	var by *string
	if build != "" {
		by = &build
	}
	for _, reference := range references {
		if reference == "" {
			continue
		}
		if _, err := i.store.Pool().Exec(ctx,
			`insert into artifact_mirrored (reference, build, why) values ($1, $2, $3)
			 on conflict (reference) do nothing`, reference, by, why); err != nil {
			return err
		}
	}
	return nil
}

// Builds is what has happened lately, newest first.
//
// For one module when named, or across the mesh when not. Both are asked: *what happened just
// now* after something goes wrong, and *what has happened to this* when deciding whether to
// trust it.
func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) {
	if limit <= 0 {
		limit = 20
	}
	query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
	          from build order by at desc limit $1`
	args := []any{limit}
	if module != "" {
		query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
		         from build where module = $2 order by at desc limit $1`
		args = append(args, module)
	}

	rows, err := i.store.Pool().Query(ctx, query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var out []Build
	for rows.Next() {
		var b Build
		var made []byte
		var asked *time.Time
		if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
			&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
			return nil, err
		}
		if asked != nil {
			b.Asked = *asked
		}
		if err := json.Unmarshal(made, &b.Made); err != nil {
			return nil, err
		}
		out = append(out, b)
	}
	return out, rows.Err()
}

// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
// repository, path, ref and commit from. False when no outcome with that id was recorded.
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
	var b Build
	var made []byte
	var asked *time.Time
	err := i.store.Pool().QueryRow(ctx,
		`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
		        coalesce(source_path,''), asked, at
		   from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
		&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
	if errors.Is(err, pgx.ErrNoRows) {
		return Build{}, false, nil
	}
	if err != nil {
		return Build{}, false, err
	}
	if asked != nil {
		b.Asked = *asked
	}
	if err := json.Unmarshal(made, &b.Made); err != nil {
		return Build{}, false, err
	}
	return b, true, nil
}

// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
//
// **The successful build of each module asked last wins**, which is the same rule the rest of the
// mesh uses for what a module currently is — asked last, not finished last (novox/hq
// 04-ISSUES/219): an older request that finishes later stood on older bases. A module rebuilt to something broken and then rebuilt again
// is at the second one; a module whose last build failed is at the last one that worked, because a
// failure published nothing and the thing it published before is still what exists.
//
// Only successes, and only builds that knew what they were building: a build that failed before it
// could read a manifest has no module to be the artifact of.
func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
	// **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280):
	// the mesh registers that build's artifacts, so a module standing on it is handed the same base
	// and is unchanged too, rather than moving for a digest an image rebuild could not help changing.
	made, err := i.heldMade(ctx)
	if err != nil {
		return nil, err
	}
	held := map[string]string{}
	for module, artifacts := range made {
		for _, artifact := range artifacts {
			if artifact.Name == "" || artifact.Reference == "" {
				continue
			}
			held[module+"/"+artifact.Name] = artifact.Reference
		}
	}
	return held, nil
}

// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select distinct on (module) module, built_against
		   from build
		  where module is not null and module <> '' and failed = ''
		  order by module, `+newestRequestFirst)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	against := map[string][]string{}
	for rows.Next() {
		var module string
		var raw []byte
		if err := rows.Scan(&module, &raw); err != nil {
			return nil, err
		}
		if len(raw) == 0 {
			continue
		}
		var refs []string
		if err := json.Unmarshal(raw, &refs); err != nil {
			continue
		}
		if len(refs) > 0 {
			against[module] = refs
		}
	}
	return against, rows.Err()
}

// ReadRepositories is what each module's newest successful build read source from besides its own
// repository, by module name.
//
// The mirror of BuiltAgainst, and derived the same way and for the same reason: a merge into a
// repository a module only packages is a change to that module, and the manifest the mesh keeps
// carries nothing that would say so (novox/hq 04-ISSUES/131).
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select distinct on (module) module, built_contexts
		   from build
		  where module is not null and module <> '' and failed = ''
		  order by module, `+newestRequestFirst)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	read := map[string][]ReadRepository{}
	for rows.Next() {
		var module string
		var raw []byte
		if err := rows.Scan(&module, &raw); err != nil {
			return nil, err
		}
		if len(raw) == 0 {
			continue
		}
		var of []ReadRepository
		if err := json.Unmarshal(raw, &of); err != nil {
			continue
		}
		if len(of) > 0 {
			read[module] = of
		}
	}
	return read, rows.Err()
}

// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
//
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
// empty declaration for a module that certainly had one.
func manifestOrNil(raw []byte) any {
	if len(raw) == 0 {
		return nil
	}
	return raw
}

// Announceable is every build worth telling a catalogue about, oldest first.
//
// **Oldest first, because a graph is built in the order things happened.** Registering a module
// that stands on a base before the base itself would make the edge point at a version the
// catalogue has not seen, and the shape of a fresh mesh guarantees that order matters: the base is
// always first and always the one that was missed.
//
// Only builds that succeeded and know what they built. A failure produced no module-version, and
// announcing one would put something in the graph that was never made — the same rule the builder
// follows when it decides whether to announce at all.
//
// One row per module and commit: a module built twice at the same commit is one fact, and the
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select distinct on (module, commit_hash)
		        id, repository, ref, module, commit_hash, built_on, failed, made,
		        source_path, manifest, built_against, at
		   from build
		  where failed = '' and module is not null and module <> '' and commit_hash <> ''
		  order by module, commit_hash, `+newestRequestFirst)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var out []Build
	for rows.Next() {
		var b Build
		var made []byte
		var manifest, against []byte
		if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
			&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
			return nil, err
		}
		if err := json.Unmarshal(made, &b.Made); err != nil {
			return nil, err
		}
		// Null rather than empty is a build recorded before the mesh kept these, and saying so is
		// the point of keeping them nullable: the replay carries nothing rather than an empty
		// declaration for a module that certainly had one.
		if len(manifest) > 0 {
			b.Manifest = manifest
		}
		if len(against) > 0 {
			if err := json.Unmarshal(against, &b.Against); err != nil {
				return nil, err
			}
		}
		out = append(out, b)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
	// the order that matters to a catalogue is a different one.
	sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
	return out, nil
}