Read a rebuild of an unchanged source as no move, whatever image digest it made (hq issue 280)
mesh/merge-gate error: the check could not run: a throwaway postgres:17-alpine could not be raised: docker run --label mesh.build=build-1791317509716888018…
mesh/delivery delivered
mesh/merge-gate error: the check could not run: a throwaway postgres:17-alpine could not be raised: docker run --label mesh.build=build-1791317509716888018…
mesh/delivery delivered
An image is not byte-reproducible, so ADR 0236's 'same artifacts is no move' never held for one: a catalogue merge that did not touch the bus rebuilt it, and every send to the control node waited for a planned bus upgrade. The builder now records a source fingerprint per build (module tree, context trees, bases and toolchains by digest). A rebuild with the fingerprint of the build it repeats is registered with that build's artifacts, handed to modules standing on it, holds no push, demands no bus step, and a plan sends and gates nothing for it. Identical artifacts remain a second way to be no move.
This commit is contained in:
@@ -67,6 +67,12 @@ type Result struct {
|
||||
// mesh keeps carries no build section, so nothing else could say that a merge there is a
|
||||
// change to this module (novox/hq 04-ISSUES/131).
|
||||
Read []catalogue.ArtifactContext
|
||||
|
||||
// Source is the build's source fingerprint (source.go): what it was made from — the module's tree,
|
||||
// the contexts' trees, the bases and toolchains by digest — hashed. Empty where the source does not
|
||||
// pin the build. Two builds with one fingerprint are one build, whatever digests they made
|
||||
// (novox/hq issue 280).
|
||||
Source string
|
||||
}
|
||||
|
||||
// GitCredential is the forge credential a clone may present when the server asks for one.
|
||||
@@ -161,6 +167,12 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
|
||||
|
||||
// What it is made from, for its source fingerprint: the module's own tree first.
|
||||
src := newSourceInputs(manifest.Module)
|
||||
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
|
||||
src.notPinned("its tree could not be named: " + err.Error())
|
||||
}
|
||||
|
||||
// A build-time credential, written into the build context as .npmrc, but ONLY for a module that
|
||||
// asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc.
|
||||
// Writing it into every context would put a per-run credential in `COPY . .` of modules that
|
||||
@@ -178,6 +190,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
|
||||
}
|
||||
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
||||
src.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
}
|
||||
|
||||
var built []catalogue.Built
|
||||
@@ -206,6 +219,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
return Result{}, err
|
||||
}
|
||||
stoodOn = bases
|
||||
src.bases = append(src.bases, bases...)
|
||||
if len(args) > 0 {
|
||||
say("bases", "%d resolved from what the mesh holds", len(args)/2)
|
||||
}
|
||||
@@ -215,7 +229,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
for _, a := range artifacts {
|
||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
|
||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
|
||||
if err != nil {
|
||||
say("artifact", "%s FAILED: %v", a.Name, err)
|
||||
return Result{}, err
|
||||
@@ -231,8 +245,20 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
return Result{}, err
|
||||
}
|
||||
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
|
||||
fingerprint := src.fingerprint()
|
||||
if fingerprint == "" {
|
||||
say("source", "no source fingerprint: %s", orNoTree(src.unpinned))
|
||||
}
|
||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest)}, nil
|
||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint}, nil
|
||||
}
|
||||
|
||||
// orNoTree is why a build has no source fingerprint, for its log.
|
||||
func orNoTree(why string) string {
|
||||
if why == "" {
|
||||
return "its tree was not named"
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
|
||||
@@ -443,7 +469,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
|
||||
|
||||
func one(ctx context.Context, run Runner, publish Publisher,
|
||||
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
|
||||
held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
|
||||
held map[string]string, npmrc string, registry Npmrc, seats map[string]string, src *sourceInputs,
|
||||
say func(step, format string, args ...any)) (catalogue.Built, error) {
|
||||
|
||||
switch a.Kind {
|
||||
@@ -525,6 +551,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
|
||||
}
|
||||
if t, err := gitTree(ctx, run, cloned, ""); err != nil {
|
||||
src.notPinned(a.Name + "'s context could not be named: " + err.Error())
|
||||
} else if src != nil {
|
||||
src.contexts[a.Name] = t
|
||||
}
|
||||
// docker build accepts -f outside the context it is given; the recipe stays exactly
|
||||
// where it was read from and validated against, absolute so the working directory
|
||||
// switching to the cloned context does not change which file that is.
|
||||
@@ -582,6 +613,14 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
"holds no copy of it. Build %s first",
|
||||
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
|
||||
}
|
||||
if src != nil {
|
||||
src.toolchains[a.Name] = toolchainOf(chain, base)
|
||||
}
|
||||
if chain.Language == "typescript" {
|
||||
if own, _ := ownDependencies(tree); len(own) > 0 {
|
||||
src.notPinned(a.Name + " resolves packages of its own at build time")
|
||||
}
|
||||
}
|
||||
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||
@@ -623,6 +662,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
// there is no Publisher call — the container itself publishes, with the credential the
|
||||
// build was handed.
|
||||
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
||||
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
||||
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
||||
|
||||
@@ -36,6 +36,8 @@ type recorded struct {
|
||||
// carried timestamps would still produce one digest — which is a test that passes for a
|
||||
// reason that has nothing to do with what it claims.
|
||||
stamped time.Time
|
||||
// tree is what git names as the tree of the module's directory; empty answers as for a commit.
|
||||
tree string
|
||||
}
|
||||
|
||||
func (r *recorded) run(_ context.Context, dir, name string, args ...string) (string, error) {
|
||||
@@ -77,6 +79,8 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
case name == "git" && len(args) > 1 && args[0] == "rev-parse" && strings.HasPrefix(args[1], "HEAD:") && r.tree != "":
|
||||
return r.tree + "\n", nil
|
||||
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
|
||||
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A build's source fingerprint: what it was made from, hashed (novox/hq issue 280).
|
||||
//
|
||||
// **An image is not byte-reproducible.** Two builds of one source make two image digests, so the rule
|
||||
// "a rebuild that made the same artifacts is no move" (novox/hq ADR 0236) held for archives and bundles
|
||||
// and never for an image: a merge that rebuilt the bus without touching it made a "new" bus build, and
|
||||
// every send to the machine running it was refused until a planned bus upgrade — for a bus nothing had
|
||||
// changed. What a build is made from is reproducible, so that is what is fingerprinted:
|
||||
//
|
||||
// - the git tree of the module's directory at the commit built — every file the build reads, its
|
||||
// module.json and its recipes among them, since the recipe and the compiler see that directory only;
|
||||
// - the git tree of each other repository an artifact's context is cloned from (ArtifactContext);
|
||||
// - each base it was handed, by digest — a module's artifact or a declared vendor image (build.on);
|
||||
// - for a bundle, the toolchain it was compiled in: the compiler image's digest and the builder's own
|
||||
// recipe for that language.
|
||||
//
|
||||
// **No fingerprint where the source does not pin the build.** A build that resolves packages from the
|
||||
// mesh's package registry at build time — a `package` artifact, a TypeScript bundle with packages of
|
||||
// its own, an image whose recipe reads the registry credential — takes whatever the registry holds
|
||||
// then, so the same source can be a different program; it records none, and only its artifacts can
|
||||
// say it is the same. A recipe that fetches from the internet without a pin is the recipe's choice
|
||||
// (novox/hq ADR 0097 refuses the unpinned bases; what a RUN step downloads is not seen here).
|
||||
|
||||
// sourcePrefix names the fingerprint's form, so a later form is never compared equal to this one.
|
||||
const sourcePrefix = "src1:"
|
||||
|
||||
// sourceInputs collects what one build was made from.
|
||||
type sourceInputs struct {
|
||||
module string
|
||||
// tree is the git tree of the module's directory at the commit built.
|
||||
tree string
|
||||
// bases are the digests of what the build was handed to stand on.
|
||||
bases []string
|
||||
// contexts are, per artifact, the git tree of the repository its context was cloned from.
|
||||
contexts map[string]string
|
||||
// toolchains are, per bundle artifact, the compiler image's digest and the recipe's hash.
|
||||
toolchains map[string]string
|
||||
// unpinned is why this build has no fingerprint: empty when it has one.
|
||||
unpinned string
|
||||
}
|
||||
|
||||
func newSourceInputs(module string) *sourceInputs {
|
||||
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}}
|
||||
}
|
||||
|
||||
// notPinned marks the build as one its source does not pin; the first reason stands.
|
||||
func (s *sourceInputs) notPinned(why string) {
|
||||
if s != nil && s.unpinned == "" {
|
||||
s.unpinned = why
|
||||
}
|
||||
}
|
||||
|
||||
// fingerprint is the build's source fingerprint, or empty when the source does not pin the build.
|
||||
func (s *sourceInputs) fingerprint() string {
|
||||
if s == nil || s.unpinned != "" || s.tree == "" {
|
||||
return ""
|
||||
}
|
||||
var lines []string
|
||||
lines = append(lines, "module "+s.module, "tree "+s.tree)
|
||||
bases := map[string]bool{}
|
||||
for _, b := range s.bases {
|
||||
bases[referenceDigest(b)] = true
|
||||
}
|
||||
for b := range bases {
|
||||
lines = append(lines, "base "+b)
|
||||
}
|
||||
for a, t := range s.contexts {
|
||||
lines = append(lines, "context "+a+" "+t)
|
||||
}
|
||||
for a, t := range s.toolchains {
|
||||
lines = append(lines, "toolchain "+a+" "+t)
|
||||
}
|
||||
// The module and its tree first, the rest in a fixed order.
|
||||
sort.Strings(lines[2:])
|
||||
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return sourcePrefix + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// referenceDigest is a reference's digest — `sha256:…` — so the registry address it was copied into does not
|
||||
// enter the fingerprint; the reference itself when it carries none.
|
||||
func referenceDigest(reference string) string {
|
||||
if _, digest, pinned := strings.Cut(reference, "@"); pinned && digest != "" {
|
||||
return digest
|
||||
}
|
||||
return reference
|
||||
}
|
||||
|
||||
// gitTree is the git tree of a directory of a clone at its checked-out commit: the whole tree for an
|
||||
// empty path.
|
||||
func gitTree(ctx context.Context, run Runner, clone, path string) (string, error) {
|
||||
spec := "HEAD^{tree}"
|
||||
if rel := strings.Trim(filepath.ToSlash(filepath.Clean(path)), "/"); path != "" && rel != "" && rel != "." {
|
||||
spec = "HEAD:" + rel
|
||||
}
|
||||
out, err := run(ctx, clone, "git", "rev-parse", spec)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tree := strings.TrimSpace(out)
|
||||
if tree == "" {
|
||||
return "", fmt.Errorf("git named no tree for %s", spec)
|
||||
}
|
||||
return tree, nil
|
||||
}
|
||||
|
||||
// toolchainOf is what a bundle's compile adds to its fingerprint: the compiler image by digest and
|
||||
// the builder's recipe for the language, hashed, so a builder that compiles differently is a change.
|
||||
func toolchainOf(chain Toolchain, base string) string {
|
||||
recipe, _ := json.Marshal(chain)
|
||||
sum := sha256.Sum256(recipe)
|
||||
return chain.Language + " " + referenceDigest(base) + " " + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build's source fingerprint (novox/hq issue 280): what it was made from, so a rebuild of an unchanged
|
||||
// source is one build however the image digest it made differs.
|
||||
|
||||
const anImage = `{"module":"bus","version":"1",
|
||||
"build":{"on":[{"arg":"BASE","image":"vendor/server@sha256:` + "1111111111111111111111111111111111111111111111111111111111111111" + `"}],
|
||||
"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]},
|
||||
"resources":[{"id":"svc","type":"container","name":"bus","artifact":"server"}]}`
|
||||
|
||||
func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string {
|
||||
t.Helper()
|
||||
r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
|
||||
r.contents["modules/bus/module.json"] = manifest
|
||||
r.tree = tree
|
||||
m := &mirroring{recorded: r, at: mirrored}
|
||||
got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
|
||||
nil, Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got.Source
|
||||
}
|
||||
|
||||
// mirroring is a store that copies a vendor's image into the mesh's registry, at an address a test says.
|
||||
type mirroring struct {
|
||||
*recorded
|
||||
at string
|
||||
}
|
||||
|
||||
func (m *mirroring) MirrorImage(_ context.Context, from, repository string) (string, error) {
|
||||
_, digest, _ := strings.Cut(from, "@")
|
||||
return m.at + "/" + repository + "@" + digest, nil
|
||||
}
|
||||
|
||||
func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) {
|
||||
one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000")
|
||||
if !strings.HasPrefix(one, sourcePrefix) {
|
||||
t.Fatalf("no fingerprint: %q", one)
|
||||
}
|
||||
// The same tree and base, the base copied to another registry address: one source.
|
||||
if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one {
|
||||
t.Fatalf("one source has two fingerprints: %s %s", one, again)
|
||||
}
|
||||
// The module's tree changed: another source.
|
||||
if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one {
|
||||
t.Fatal("a changed tree kept its fingerprint")
|
||||
}
|
||||
// The base moved: another source.
|
||||
moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"2222222222222222222222222222222222222222222222222222222222222222", 1)
|
||||
if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one {
|
||||
t.Fatal("a moved base kept its fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildTheRegistryDecidesHasNoFingerprint(t *testing.T) {
|
||||
s := newSourceInputs("app")
|
||||
s.tree = "aaaa"
|
||||
if s.fingerprint() == "" {
|
||||
t.Fatal("a pinned source has no fingerprint")
|
||||
}
|
||||
s.notPinned("it resolves packages from the mesh's registry at build time")
|
||||
if got := s.fingerprint(); got != "" {
|
||||
t.Fatalf("a build the registry decides has a fingerprint: %s", got)
|
||||
}
|
||||
if (&sourceInputs{module: "app"}).fingerprint() != "" {
|
||||
t.Fatal("a build whose tree was not named has a fingerprint")
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,9 @@ type Build struct {
|
||||
// Read is every repository this build read source from besides the module's own (novox/hq
|
||||
// 04-ISSUES/131), at the ref it read.
|
||||
Read []ReadRepository
|
||||
// SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq
|
||||
// issue 280); empty from a builder that predates it, or where the source does not pin the build.
|
||||
SourceFingerprint string
|
||||
// Failed is the builder's own words, empty when it worked.
|
||||
Failed string
|
||||
Made []Artifact
|
||||
@@ -112,11 +115,11 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
|
||||
source_path, manifest, built_against, built_contexts, asked)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
source_path, manifest, built_against, built_contexts, asked, source_fingerprint)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
on conflict (id) do nothing`,
|
||||
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
|
||||
b.Path, manifestOrNil(b.Manifest), against, read, asked)
|
||||
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -202,38 +205,23 @@ func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, erro
|
||||
// Only successes, and only builds that knew what they were building: a build that failed before it
|
||||
// could read a manifest has no module to be the artifact of.
|
||||
func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select distinct on (module) module, made
|
||||
from build
|
||||
where module is not null and module <> '' and failed = ''
|
||||
order by module, `+newestRequestFirst)
|
||||
// **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280):
|
||||
// the mesh registers that build's artifacts, so a module standing on it is handed the same base
|
||||
// and is unchanged too, rather than moving for a digest an image rebuild could not help changing.
|
||||
made, err := i.heldMade(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
held := map[string]string{}
|
||||
for rows.Next() {
|
||||
var module string
|
||||
var raw []byte
|
||||
if err := rows.Scan(&module, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var made []Artifact
|
||||
if err := json.Unmarshal(raw, &made); err != nil {
|
||||
// Skipped rather than fatal. One unreadable build record should not stop every other
|
||||
// module's base from being answerable — and the build that needs this one will say
|
||||
// plainly that it is missing.
|
||||
continue
|
||||
}
|
||||
for _, artifact := range made {
|
||||
for module, artifacts := range made {
|
||||
for _, artifact := range artifacts {
|
||||
if artifact.Name == "" || artifact.Reference == "" {
|
||||
continue
|
||||
}
|
||||
held[module+"/"+artifact.Name] = artifact.Reference
|
||||
}
|
||||
}
|
||||
return held, rows.Err()
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-- A build says what it was made from (novox/hq issue 280).
|
||||
--
|
||||
-- A rebuild was "no move" only when it made the same artifacts (novox/hq ADR 0236), and an image is not
|
||||
-- byte-reproducible: a merge that rebuilt the bus without touching its source made a new bus image
|
||||
-- digest, the mesh read it as a new bus build, and every send to the machine running the bus was
|
||||
-- refused until a planned bus upgrade. The builder now says what the build was made from — the git
|
||||
-- tree of the module's directory, the trees of the contexts it read, its bases and toolchains by
|
||||
-- digest — hashed, and two builds with one source fingerprint are one build.
|
||||
--
|
||||
-- Empty for every build recorded before this and for a build whose source does not pin it (one that
|
||||
-- resolves packages from the registry at build time): those are told apart by their artifacts alone,
|
||||
-- exactly as before.
|
||||
alter table build add column source_fingerprint text not null default '';
|
||||
@@ -0,0 +1,186 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// A build whose source is unchanged is never a move (novox/hq issue 280).
|
||||
//
|
||||
// The builder says what each build was made from, hashed (its source fingerprint). Two successful
|
||||
// builds of a module with one fingerprint are one build, whatever digests they made — an image is not
|
||||
// byte-reproducible, and reading a rebuild's new image digest as a new build made a merge that never
|
||||
// touched the bus demand a planned bus upgrade before anything could be sent to the machine running
|
||||
// it. So:
|
||||
//
|
||||
// - a module's builds, newest request first, fall into runs of one fingerprint; **the oldest build of
|
||||
// the newest run stands for the run**: its artifacts are the ones the mesh registers and hands
|
||||
// every module that stands on it (Held), so a rebuild of an unchanged source changes nothing any
|
||||
// machine is sent, and nothing standing on it moves either;
|
||||
// - a build that failed its gate ends a run: what was put back is never what a later build stands for;
|
||||
// - an empty fingerprint — a builder that predates it, a source that does not pin its build — is a
|
||||
// run of its own, as every build was before.
|
||||
|
||||
// sourceRow is one successful build of a module, as the runs are read.
|
||||
type sourceRow struct {
|
||||
id, commit, fingerprint string
|
||||
made []byte
|
||||
manifest []byte
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// sourceRows is every successful build of each module (of one module, when named), newest request
|
||||
// first.
|
||||
func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) {
|
||||
query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest,
|
||||
coalesce(g.verdict = 'failed', false)
|
||||
from build b left join build_gate g on g.build = b.id
|
||||
where b.module is not null and b.module <> '' and b.failed = ''`
|
||||
args := []any{}
|
||||
if module != "" {
|
||||
query += ` and b.module = $1`
|
||||
args = append(args, module)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string][]sourceRow{}
|
||||
var order []string
|
||||
for rows.Next() {
|
||||
var m string
|
||||
var r sourceRow
|
||||
if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if _, seen := out[m]; !seen {
|
||||
order = append(order, m)
|
||||
}
|
||||
out[m] = append(out[m], r)
|
||||
}
|
||||
return out, order, rows.Err()
|
||||
}
|
||||
|
||||
// standing is, of a module's builds newest first, the index of the build that stands for the build at
|
||||
// `from`: the oldest of the unbroken run of builds behind it with its source fingerprint.
|
||||
func standing(builds []sourceRow, from int) int {
|
||||
at := from
|
||||
fp := builds[from].fingerprint
|
||||
if fp == "" || builds[from].failedGate {
|
||||
return at
|
||||
}
|
||||
for j := from + 1; j < len(builds); j++ {
|
||||
if builds[j].fingerprint != fp || builds[j].failedGate {
|
||||
break
|
||||
}
|
||||
at = j
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// StandingBuild is the build that stands for a module's build (novox/hq issue 280): the oldest build of
|
||||
// the unbroken run of builds with its source fingerprint, at or before it — itself when its fingerprint
|
||||
// is empty or it starts the run. Answers its id and the manifest it was recorded with; empty when the
|
||||
// build is not a successful build of the module on record.
|
||||
func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (string, []byte, error) {
|
||||
all, _, err := i.sourceRows(ctx, module)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
builds := all[module]
|
||||
for k, b := range builds {
|
||||
if b.id == build {
|
||||
s := builds[standing(builds, k)]
|
||||
return s.id, s.manifest, nil
|
||||
}
|
||||
}
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
// SourceFingerprints is, per module, per commit, the source fingerprint of the newest successful build
|
||||
// from it that has one: module → commit → fingerprint. A commit whose builds carry none is absent.
|
||||
func (i *Inventory) SourceFingerprints(ctx context.Context) (map[string]map[string]string, error) {
|
||||
all, _, err := i.sourceRows(ctx, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for m, builds := range all {
|
||||
for _, b := range builds {
|
||||
if b.fingerprint == "" || b.commit == "" {
|
||||
continue
|
||||
}
|
||||
if out[m] == nil {
|
||||
out[m] = map[string]string{}
|
||||
}
|
||||
if _, seen := out[m][b.commit]; !seen {
|
||||
out[m][b.commit] = b.fingerprint
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BuildSourceFingerprints is SourceFingerprints for one module: commit → fingerprint.
|
||||
func (i *Inventory) BuildSourceFingerprints(ctx context.Context, module string) (map[string]string, error) {
|
||||
all, err := i.SourceFingerprints(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if all[module] == nil {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
return all[module], nil
|
||||
}
|
||||
|
||||
// heldMade is, per module, what the build standing for its newest successful build made — what Held
|
||||
// answers (novox/hq issue 280).
|
||||
func (i *Inventory) heldMade(ctx context.Context) (map[string][]Artifact, error) {
|
||||
all, _, err := i.sourceRows(ctx, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]Artifact{}
|
||||
for m, builds := range all {
|
||||
if len(builds) == 0 {
|
||||
continue
|
||||
}
|
||||
var made []Artifact
|
||||
if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err != nil {
|
||||
// Skipped rather than fatal, as Held always did: the build that needs it says it is missing.
|
||||
continue
|
||||
}
|
||||
out[m] = made
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SameSourceCommits is the commits of the builds in a build's run (novox/hq issue 280): the build and
|
||||
// every build behind it back to the one standing for it, all made from one source. Empty when the
|
||||
// build stands for itself — no earlier build was made from its source — so a commit alone never says
|
||||
// two builds are one: a dependent rebuilt because its base moved keeps its commit and is a move.
|
||||
func (i *Inventory) SameSourceCommits(ctx context.Context, module, build string) (map[string]bool, error) {
|
||||
all, _, err := i.sourceRows(ctx, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := all[module]
|
||||
for k, b := range builds {
|
||||
if b.id != build {
|
||||
continue
|
||||
}
|
||||
s := standing(builds, k)
|
||||
if s == k {
|
||||
return nil, nil
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for j := k; j <= s; j++ {
|
||||
if builds[j].commit != "" {
|
||||
out[builds[j].commit] = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -179,6 +179,13 @@ type BuildResult struct {
|
||||
// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
|
||||
Read []ReadRepository `json:"read,omitempty"`
|
||||
|
||||
// SourceFingerprint is what the build was made from, hashed (novox/hq issue 280): the module's
|
||||
// tree at the commit, the trees of the contexts it read, its bases and toolchains by digest. Two
|
||||
// builds with one fingerprint are one build, however their digests differ — an image is not
|
||||
// byte-reproducible. Empty from a builder that predates it, or where the source does not pin the
|
||||
// build; then only identical artifacts make a rebuild no move.
|
||||
SourceFingerprint string `json:"source-fingerprint,omitempty"`
|
||||
|
||||
// Failed is why, when it did.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
|
||||
|
||||
Reference in New Issue
Block a user