mesh-controller/internal/link/build.go

package link

import (
	"encoding/json"
	"strconv"
	"strings"
	"time"
)

// Asking a machine to build a module, and hearing what came out.
//
// **A build is work, not state.** Everything else the control plane sends a node is a declaration
// — *this is what you should be* — and the node reconciles toward it forever. A build happens
// once, produces something, and is finished. Putting it in a declaration would mean a machine
// rebuilding on every reconcile, or the declaration carrying "and I already did this", which is
// state about an event rather than about the machine.
//
// So it travels on its own queue, and the reply comes back correlated. That is also why the
// builder is a **separate consumer** rather than the host: the host applies declarations and
// holds no opinion about what they contain, and a host that also built things would be a host
// with a container runtime requirement and a git dependency (novox/hq ADR 0005).

// NewBuildID is the correlation for a build asked at that moment: `build-<unix nanoseconds>`.
//
// **The id carries when the build was asked, and that is read back** (novox/hq 04-ISSUES/219). Builds
// of one module can be in flight together and finish in any order; what a module currently is must
// be the newest *request's* outcome, not the last one heard, and the id is the one thing every
// outcome echoes whichever builder answered it. One place writes the shape and one reads it.
func NewBuildID(asked time.Time) string {
	return "build-" + strconv.FormatInt(asked.UnixNano(), 10)
}

// BuildAskedAt is when the build with this id was asked, as NewBuildID wrote it. False for an id
// of any other shape — one written before this was read, or by hand — whose request time the mesh
// does not know.
func BuildAskedAt(id string) (time.Time, bool) {
	digits, ok := strings.CutPrefix(id, "build-")
	if !ok || digits == "" {
		return time.Time{}, false
	}
	nanos, err := strconv.ParseInt(digits, 10, 64)
	// A number too small to be a moment this mesh could have asked at is a name, not a time.
	if err != nil || nanos < time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano() {
		return time.Time{}, false
	}
	return time.Unix(0, nanos).UTC(), true
}

// BuildRequest is one module to build.
type BuildRequest struct {
	// ID correlates the answer with the asking. Not the module name: two builds of one module can
	// be in flight, and the second answer is not the first one's.
	ID string `json:"id"`
	// Repository is where the source is, as git would clone it.
	Repository string `json:"repository"`
	// Ref is the branch, tag or commit. Empty means whatever the repository's default is, which
	// is the only case where the mesh does not know what it built until it has built it.
	Ref string `json:"ref,omitempty"`
	// Path is the module's directory inside that repository (novox/hq ADR 0069). Empty means the
	// repository root, which is the ordinary case; a repository holding several modules names
	// each by its own directory.
	Path string `json:"path,omitempty"`
	// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
	//
	// **Sent with the asking rather than fetched by the builder** (novox/hq issue 044). A module
	// says which module's artifact its build stands on; only the mesh knows which copy of that
	// artifact *this* mesh holds, and the builder is deliberately a thing that clones, runs a
	// build and answers — giving it a way to ask the mesh questions would make it something else.
	// So the answer travels with the question.
	//
	// It is everything rather than only what this module needs, because what this module needs is
	// written in a manifest the mesh has not read: it is inside the repository, and reading it is
	// the build's first act.
	Held map[string]string `json:"held,omitempty"`
	// Seats is the clone base — `scheme://host:port` — of each seat a recipe's context may name
	// (novox/hq ADR 0155): `git` for this mesh's own forge. Sent with the asking for the reason
	// Held is: the context is written in a manifest the mesh has not read, and only the mesh knows
	// which forge holds the seat here. A builder handed no base for a seat a context names refuses
	// the build and says so.
	Seats map[string]string `json:"seats,omitempty"`
	// Source is the repository as the mesh records it when it lives on a seat's holder — the seat
	// and the path on it, never the URL just composed (novox/hq ADR 0111). Carried with the
	// asking and echoed in the outcome, so whoever hears the outcome can register the module with
	// its true source, whether or not they were the one who asked (novox/hq issue 176).
	Source *SourceOnSeat `json:"source,omitempty"`
	// DryRun says the asker wants the outcome to look at and nothing else (novox/hq issue 240): the
	// builder echoes it, and whoever hears the outcome takes nothing in — no record, no registration,
	// no plan, nothing a push could send.
	DryRun bool `json:"dry-run,omitempty"`
	// Check makes this ask a pull request's merge check rather than a build (novox/hq to-be 45 §9): the
	// builder checks the repository out at Ref with the repositories it is checked beside, reads the
	// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
	// merge-check.sh and answers its verdict. Nothing is built, published or registered.
	Check *CheckRequest `json:"check,omitempty"`
	// Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key
	// of the commit's tree and, where a check build holds the same key, copies it by digest into the module's
	// namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a
	// differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never
	// takes a check's build. A builder from before it ignores it and builds.
	Reuse *ReuseAsk `json:"reuse,omitempty"`
}

// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2).
//
// **Only a build a verdict names is taken.** Builds are the builds the build seat said, in the outcomes of its checks
// of the walk's own pull requests, that it made for their merges: each key with its artifacts' digests. The record
// in the scratch namespace is not trusted alone — the artifact store takes writes from anything on the network, a
// check runs a pull request's own script on that network, and a record planted under another merge's key would
// otherwise be registered. The builder takes a scratch build only when a build here names its key and every digest.
type ReuseAsk struct {
	Pulls  []PullRef      `json:"pulls,omitempty"`
	Builds []CheckedBuild `json:"builds,omitempty"`
}

// CheckedBuild is one build a check's verdict names: the pull request it checked, its ask, and the build.
type CheckedBuild struct {
	Pull  PullRef `json:"pull"`
	Check string  `json:"check"`
	// Head is the pull request's head the check judged, as its verdict says it (twelve characters): a build is
	// taken or kept only by a check of the same head (novox/hq issue 703).
	Head  string     `json:"head,omitempty"`
	Built CheckBuilt `json:"built"`
}

// SameHead says two spellings of a commit name the same one: both said, one the other's beginning, at least
// twelve characters of it (a verdict says twelve).
func SameHead(a, b string) bool {
	if len(a) > len(b) {
		a, b = b, a
	}
	return len(a) >= 12 && strings.HasPrefix(b, a)
}

// PullRef names a pull request.
type PullRef struct {
	Owner  string `json:"owner"`
	Repo   string `json:"repo"`
	Number int    `json:"number"`
}

// CheckRequest is what a merge check needs beyond the repository and its head.
type CheckRequest struct {
	Owner  string `json:"owner"`
	Repo   string `json:"repo"`
	Number int    `json:"number,omitempty"`
	Base   string `json:"base,omitempty"`
	// Paths are the files the pull request changes, for the width of its rebuild.
	Paths []string `json:"paths,omitempty"`
	// Beside are the repositories the check reads next to this one, each cloned at the ref given — the
	// controller the mesh runs, the catalogue it holds, the host it runs — keyed by the directory name the
	// check finds it under.
	Beside map[string]CheckedOut `json:"beside,omitempty"`
	// Modules are the modules of the mesh's graph the change touches, and New the directories it adds a
	// module in that the graph does not hold (novox/hq ADR 0237 as amended): **the graph decides whether
	// the gate runs**, not the repository. With neither, only the repository's own merge-check.sh runs.
	Modules []string `json:"modules,omitempty"`
	New     []string `json:"new,omitempty"`
	// Dependents are the modules a merge would build after Modules because they stand on them: the
	// planner's dependency walk, said on the pull request.
	Dependents []string `json:"dependents,omitempty"`
	// Plan is the change plan of the commit checked, computed by the controller and echoed with the outcome.
	Plan *ChangePlan `json:"plan,omitempty"`
	// Manifests are the touched modules' manifests in the change's tree, by path from its root: what the
	// gate puts through `module check`.
	Manifests []string `json:"manifests,omitempty"`
	// Judge is who judges the gate: empty for the controller the mesh runs; JudgeSelf for a change to
	// the controller, judged by itself; JudgeValidator for a change to the node-engine, judged by the
	// running controller built with the change's validator in place of the one it vendors.
	Judge string `json:"judge,omitempty"`
	// Release is the release branch the pull request merges into (novox/hq ADR 0358): its base, when that is not its
	// repository's trunk and mesh-controller has a branch of that name. The gate is then judged by the controller
	// built from mesh-controller's release branch — a change to the controller still by itself, a change to the
	// node-engine with its own validator — and each repository beside it is read at its branch of that name where it
	// has one (CheckedOut.Else otherwise). Empty for every other base, checked as before it.
	Release string `json:"release,omitempty"`
	// Group names a delivery group whose members' heads are composed together with this one (novox/hq ADR
	// 0239), and Members are those other heads, each cloned beside it and laid over the mesh in turn. The
	// repository's own check is not run for a group: each member's pull request runs its own.
	Group   string        `json:"group,omitempty"`
	Members []GroupMember `json:"members,omitempty"`
	// Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds
	// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
	// take. A build seat from before it builds nothing, and the walk builds as before.
	Builds []CheckBuild `json:"builds,omitempty"`
	// Running is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module — the
	// build its machines are sent (novox/hq issue 721): what a build for the merge is compared with, never the newest
	// build, which may be one its gate failed and the mesh put back. A module absent is compared with nothing, and
	// moves. A controller from before it sends none, and nothing is said unchanged.
	Running map[string]string `json:"running,omitempty"`
	// Named are the builds the seat's verdicts of earlier checks of this pull request at this head name, within the
	// week check builds are kept (novox/hq issue 703): asked again because only the trunk moved, the check keeps a
	// build kept under a module's unchanged key, instead of building it again, only when one of these names its key
	// and every digest. Never another pull request's, nor an earlier head's. A build seat from before it builds.
	Named []CheckedBuild `json:"named,omitempty"`
	// Try marks a branch tried before its pull request (novox/hq issue 686): the repository's own check for a
	// pushed branch or commit, asked by a builder through the controller's `try` verb. Its outcome is read from
	// the build's log (StepTried) and said to nobody: no status on the forge, nothing recorded or registered,
	// nothing built for a merge. Tests narrows what the repository's script runs to the tests a pattern names,
	// and StoreOnly to its store tests; a script that reads neither runs whole.
	Try       bool   `json:"try,omitempty"`
	Tests     string `json:"tests,omitempty"`
	StoreOnly bool   `json:"store-only,omitempty"`
}

// StepTried is the step a build seat says a try's outcome under, in the build's log: one line, a Tried as JSON
// (novox/hq issue 686).
const StepTried = "tried"

// Tried is what came of a branch tried before its pull request (novox/hq issue 686): the verdict of the
// repository's own check for the ref merged into the trunk, how long it took, and the lines that name what
// failed, picked from the whole of its output (issue 460).
type Tried struct {
	Verdict string   `json:"verdict"`
	Summary string   `json:"summary"`
	Took    string   `json:"took"`
	Head    string   `json:"head,omitempty"`
	Trunk   string   `json:"trunk,omitempty"`
	Failed  []string `json:"failed,omitempty"`
}

// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would.
type CheckBuild struct {
	Module     string `json:"module"`
	Repository string `json:"repository,omitempty"`
	Path       string `json:"path,omitempty"`
	Ref        string `json:"ref,omitempty"`
	Merged     bool   `json:"merged,omitempty"`
	Tier       int    `json:"tier"`
	// Skip says why the check cannot build it as the walk would — a module that reads the repository checked
	// through a context, which a check cannot hand the merge — and nothing after it is built either.
	Skip string `json:"skip,omitempty"`
}

// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key
// and the artifacts, in the scratch namespace — or why it was not built.
type CheckBuilt struct {
	Module   string         `json:"module"`
	Key      string         `json:"key,omitempty"`
	Made     []MadeArtifact `json:"made,omitempty"`
	NotBuilt string         `json:"not-built,omitempty"`
	Took     string         `json:"took,omitempty"`
	// Kept names the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
	// a walk takes it as that check's build. Empty when this check built it.
	Kept string `json:"kept,omitempty"`
	// Same says every artifact came out the same as the build the mesh holds of the module, and SameAs the held
	// reference each was compared with (novox/hq issue 721): a walk cut while the mesh still holds those builds leaves
	// the module out. Empty when it differs, or could not be compared.
	Same   string            `json:"same,omitempty"`
	SameAs map[string]string `json:"same-as,omitempty"`
}

// GroupMember is one other head of a delivery group's composed check.
type GroupMember struct {
	Owner      string   `json:"owner"`
	Repo       string   `json:"repo"`
	Number     int      `json:"number,omitempty"`
	Repository string   `json:"repository"`
	Ref        string   `json:"ref"`
	Paths      []string `json:"paths,omitempty"`
	// Base is the branch it merges into: the build seat merges the head into that trunk's head before the
	// group is composed (novox/hq ADR 0323 §6). A build seat from before it composes the head alone.
	Base string `json:"base,omitempty"`
}

// Who judges a merge check's gate.
const (
	JudgeSelf      = "self"
	JudgeValidator = "validator"
)

// CheckedOut is a repository cloned beside a check, at a ref.
type CheckedOut struct {
	Repository string `json:"repository"`
	Ref        string `json:"ref,omitempty"`
	// Else is the ref read when the repository has no branch Ref (novox/hq ADR 0358 §3): beside a check of a pull
	// request into a release branch, Ref is that branch and Else the repository's trunk. Empty: Ref is read, or the
	// check cannot run.
	Else string `json:"else,omitempty"`
}

// CheckOutcome is a merge check's verdict.
type CheckOutcome struct {
	// Verdict is pass, warning, fail, or error: the check could not run, which is never a pass.
	Verdict string `json:"verdict"`
	Summary string `json:"summary"`
	// Report is the check's own account, its last lines, bounded.
	Report string `json:"report,omitempty"`
	// Took is how long it ran.
	Took string `json:"took,omitempty"`
	// Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's.
	Gate      *CheckLayer `json:"gate,omitempty"`
	RepoCheck *CheckLayer `json:"repo-check,omitempty"`
	// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
	// commit, each as twelve characters. Trunk is empty from a build seat from before it, which judged the head
	// alone. Beside are the commits of the repositories read beside it, by directory, named and not judged
	// against (issue 568); Members a delivery group's other heads, each with the trunk commit it merged into.
	Head    string            `json:"head,omitempty"`
	Trunk   string            `json:"trunk,omitempty"`
	Beside  map[string]string `json:"beside,omitempty"`
	Members []CheckedMember   `json:"members,omitempty"`
	// Built are the builds it made for its merge (novox/hq ADR 0350 §1).
	Built []CheckBuilt `json:"built,omitempty"`
	// Release is the release branch the check was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the
	// controller that judged its gate, as twelve characters, and BesideAt the branch each repository beside it was
	// read at, by directory. Empty from a check of any other base, and from a build seat from before it.
	Release  string            `json:"release,omitempty"`
	JudgedBy string            `json:"judged-by,omitempty"`
	BesideAt map[string]string `json:"beside-at,omitempty"`
	// Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2).
	Validator string `json:"validator,omitempty"`
}

// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
type SourceOnSeat struct {
	Seat       string `json:"seat"`
	Repository string `json:"repository"`
}

// BuildResult is what a builder says back.
//
// **Failure is a result, not an absence.** A build that fails and says nothing is
// indistinguishable from a builder that is not running, and those want completely different
// responses — the same rule the host follows about a service that does not exist.
type BuildResult struct {
	ID         string `json:"id"`
	Repository string `json:"repository"`
	// Path is echoed back, so what the mesh records as this module's source is what was actually
	// built rather than what the asker meant (novox/hq ADR 0069).
	Path string `json:"path,omitempty"`
	Ref  string `json:"ref,omitempty"`

	// On is the machine that did it, so a failure that is about one machine can be told from one
	// about the source.
	On string `json:"on"`

	// Module is what was built, read out of the manifest — the only place it is authoritative, since
	// a request names a repository and a path.
	//
	// **Here because one message now reaches three audiences** (novox/hq ADR 0121). On the bus the
	// mesh runs on today the answer and the announcement were two publishes to two topologies, so a
	// result needed no module name and the announcement carried one. On the bus being built the
	// outcome is the role's own event, and the catalogue reading it needs to know what was built.
	// Empty on a failed build, which produced no module version.
	Module string `json:"module,omitempty"`

	// Commit is what was actually built. The mesh records it, which is what makes "is this
	// current?" answerable without building again.
	Commit string `json:"commit,omitempty"`

	// Manifest is the module as the mesh should hold it, artifacts resolved to digests. Raw,
	// because the control plane parses it with the same parser it uses for one handed over by
	// hand — a second path would be a second thing to disagree.
	Manifest json.RawMessage `json:"manifest,omitempty"`

	// Made is each artifact, for reporting.
	Made []MadeArtifact `json:"made,omitempty"`

	// Against is every pinned image this was built on top of, read out of the build's own inputs
	// (novox/hq ADR 0009). The catalogue turns these into edges; nothing else need care.
	Against []string `json:"against,omitempty"`

	// Mirrored is every base this build copied into the artifact store, as pinned (novox/hq ADR
	// 0257). Carried on a failed build too: the copy is in the store whether the build worked or not.
	Mirrored []string `json:"mirrored,omitempty"`

	// Read is every repository this build read source from besides the module's own. A module whose
	// recipe packages source that lives elsewhere is affected when that repository moves, and the
	// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
	Read []ReadRepository `json:"read,omitempty"`

	// Sources are what the build was made from, as files, per repository (novox/hq ADR 0267): the
	// entries the planner maps the next merge's changed files onto. Said only for a build of a commit on
	// the trunk, and only for a repository whose every artifact's build source the builder knows; a
	// repository it says nothing of is read whole, as before. Empty from a builder that predates it.
	Sources []BuildSource `json:"sources,omitempty"`

	// Trunk is the repository's default branch at the build, and OnTrunk whether the commit built is on it
	// (novox/hq ADR 0238): **only a commit on the trunk is published** — the controller refuses to register
	// a build of one off it. Empty Trunk is a build seat that could not say, or predates the rule.
	Trunk   string `json:"trunk,omitempty"`
	OnTrunk bool   `json:"on-trunk,omitempty"`
	// Branches are every branch the commit is on: the trunk of a module that follows a branch other than
	// the repository's default is the branch it follows.
	Branches []string `json:"branches,omitempty"`

	// SourceFingerprint is what the build was made from, hashed (novox/hq issue 280): the module's
	// tree at the commit, the trees of the contexts it read, its bases and toolchains by digest. Two
	// builds with one fingerprint are one build, however their digests differ — an image is not
	// byte-reproducible. Empty from a builder that predates it, or where the source does not pin the
	// build; then only identical artifacts make a rebuild no move.
	SourceFingerprint string `json:"source-fingerprint,omitempty"`

	// Delivery is the how-manyth time the queue handed this ask out (BuildStart.Delivery): more than 1
	// says the outcome is from a run started again after an earlier taker left it (novox/hq issue 594).
	Delivery int `json:"delivery,omitempty"`

	// Failed is why, when it did.
	Failed string `json:"failed,omitempty"`

	// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build
	// instead of building, is "reused from <check build>" with its digests, and ReusedFrom which; BuiltWhy, on one
	// that built, why it took none (§3, §4). Both empty from a builder that predates it.
	ReuseKey   string      `json:"reuse-key,omitempty"`
	Reused     string      `json:"reused,omitempty"`
	ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
	BuiltWhy   string      `json:"built-why,omitempty"`

	// Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR
	// 0348 §7), which the manifest was judged against: the controller publishes the catalogue's from a trunk
	// build, the newest by Generation winning. Absent for a repository holding none, and from a builder that
	// predates the list.
	Predefined *PredefinedList `json:"predefined,omitempty"`

	// Source is the request's, echoed: the seat form of the repository, for whoever registers.
	Source *SourceOnSeat `json:"source,omitempty"`

	// DryRun is the request's, echoed: an outcome nobody may take in (novox/hq issue 240).
	DryRun bool `json:"dry-run,omitempty"`

	// Check is a merge check's verdict, for an ask that was one; the request's Check is echoed in
	// Checked so whoever hears it knows which pull request it judged.
	Check   *CheckOutcome `json:"check-outcome,omitempty"`
	Checked *CheckRequest `json:"check,omitempty"`
}

// PredefinedList is the list of predefined keys a build read (novox/hq ADR 0348 §7): Keys its canonical text, a
// JSON object of keys, and Generation how many commits the commit built stands on — of two commits on one trunk
// the later has more, which is what orders two builds' lists; 0 when the builder could not say.
type PredefinedList struct {
	Keys       json.RawMessage `json:"keys"`
	Generation int             `json:"generation,omitempty"`
}

// ReadRepository is a repository a build read source from besides the module's own, at the branch,
// tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason
// MadeArtifact is: one direction of dependency.
type ReadRepository struct {
	Repository string `json:"repository"`
	Ref        string `json:"ref,omitempty"`
}

// BuildSource is the build source a build read in one repository (novox/hq ADR 0267): Repository and Ref
// a context's, as its manifest names it, and empty for the module's own; Paths the entries, relative to
// that repository's root — `dir/` a Go package's directory but its tests, `dir/**` everything below a
// directory, `**` the whole tree, anything else one file (builder.SourceHolds).
type BuildSource struct {
	Repository string   `json:"repository,omitempty"`
	Ref        string   `json:"ref,omitempty"`
	Paths      []string `json:"paths"`
}

// MadeArtifact is one thing a build produced, as a person would want it reported.
type MadeArtifact struct {
	Name      string `json:"name"`
	Kind      string `json:"kind"`
	Reference string `json:"reference"`
}

// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk
// commit the check merged it into, the check's ask, and its key.
type ReusedFrom struct {
	Pull  PullRef `json:"pull"`
	Head  string  `json:"head,omitempty"`
	Trunk string  `json:"trunk,omitempty"`
	Check string  `json:"check,omitempty"`
	Key   string  `json:"key"`
}

// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from <check build>", or "built:
// <why>". A build seat from before it says neither, and that is said.
func (r BuildResult) MadeWords() string {
	switch {
	case r.Reused != "":
		return r.Reused
	case r.BuiltWhy != "":
		return "built: " + r.BuiltWhy
	}
	return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)"
}