jochen
abe5f7dc17
A hand build of an older trunk commit said a closure lacking what was imported since, and the planner would have mapped the next merge onto it. C and assembly beside Go may include files below their directory, and a go.mod made above a package moves it out of its module: each is now held.
269 lines
8.7 KiB
Go
269 lines
8.7 KiB
Go
package builder
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
)
|
|
|
|
// A build's source fingerprint: what it was made from, hashed (novox/hq issue 280).
|
|
//
|
|
// **An image is not byte-reproducible.** Two builds of one source make two image digests, so the rule
|
|
// "a rebuild that made the same artifacts is no move" (novox/hq ADR 0236) held for archives and bundles
|
|
// and never for an image: a merge that rebuilt the bus without touching it made a "new" bus build, and
|
|
// every send to the machine running it was refused until a planned bus upgrade — for a bus nothing had
|
|
// changed. What a build is made from is reproducible, so that is what is fingerprinted:
|
|
//
|
|
// - the git tree of the module's directory at the commit built — every file the build reads, its
|
|
// module.json and its recipes among them, since the recipe and the compiler see that directory only;
|
|
// - the git tree of each other repository an artifact's context is cloned from (ArtifactContext);
|
|
// - each base it was handed, by digest — a module's artifact or a declared vendor image (build.on);
|
|
// - for a bundle, the toolchain it was compiled in: the compiler image's digest and the builder's own
|
|
// recipe for that language.
|
|
//
|
|
// **No fingerprint where the source does not pin the build.** A build that resolves packages from the
|
|
// mesh's package registry at build time — a `package` artifact, a TypeScript bundle with packages of
|
|
// its own, an image whose recipe reads the registry credential — takes whatever the registry holds
|
|
// then, so the same source can be a different program; it records none, and only its artifacts can
|
|
// say it is the same. A recipe that fetches from the internet without a pin is the recipe's choice
|
|
// (novox/hq ADR 0097 refuses the unpinned bases; what a RUN step downloads is not seen here).
|
|
|
|
// sourcePrefix names the fingerprint's form, so a later form is never compared equal to this one.
|
|
const sourcePrefix = "src1:"
|
|
|
|
// sourceInputs collects what one build was made from.
|
|
type sourceInputs struct {
|
|
module string
|
|
// tree is the git tree of the module's directory at the commit built.
|
|
tree string
|
|
// bases are the digests of what the build was handed to stand on.
|
|
bases []string
|
|
// contexts are, per artifact, the git tree of the repository its context was cloned from.
|
|
contexts map[string]string
|
|
// toolchains are, per bundle artifact, the compiler image's digest and the recipe's hash.
|
|
toolchains map[string]string
|
|
// unpinned is why this build has no fingerprint: empty when it has one.
|
|
unpinned string
|
|
|
|
// prefix is the module's directory within its repository, empty at the root.
|
|
prefix string
|
|
// own is the module's build source in its own repository, relative to the module's directory, and
|
|
// whole when an artifact's is not known (novox/hq ADR 0267).
|
|
own map[string]bool
|
|
ownIsAll bool
|
|
// read is, per context (repository and ref), the build source read there; nil for one read whole.
|
|
read map[string]map[string]bool
|
|
readAs map[string]catalogue.ArtifactContext
|
|
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
|
// closure is not the one the next merge there meets, and the build says no build source.
|
|
contextsAtHead bool
|
|
}
|
|
|
|
// contextOffHead says a context was not its trunk's head.
|
|
func (s *sourceInputs) contextOffHead() {
|
|
if s != nil {
|
|
s.contextsAtHead = false
|
|
}
|
|
}
|
|
|
|
func newSourceInputs(module string) *sourceInputs {
|
|
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{},
|
|
own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
|
|
contextsAtHead: true}
|
|
}
|
|
|
|
// ownHas adds entries, relative to the module's directory, to its build source in its own repository.
|
|
func (s *sourceInputs) ownHas(entries ...string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
s.own[e] = true
|
|
}
|
|
}
|
|
|
|
// ownWhole says an artifact's build source in the module's own repository is not known: the module's
|
|
// whole directory is its source, as it was before.
|
|
func (s *sourceInputs) ownWhole() {
|
|
if s != nil {
|
|
s.ownIsAll = true
|
|
}
|
|
}
|
|
|
|
func contextKey(c catalogue.ArtifactContext) string { return c.Repository + "#" + c.Ref }
|
|
|
|
// readIn adds entries to the build source read in a context; one read whole stays whole.
|
|
func (s *sourceInputs) readIn(c catalogue.ArtifactContext, entries []string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
key := contextKey(c)
|
|
s.readAs[key] = c
|
|
set, known := s.read[key]
|
|
if known && set == nil {
|
|
return
|
|
}
|
|
if set == nil {
|
|
set = map[string]bool{}
|
|
s.read[key] = set
|
|
}
|
|
for _, e := range entries {
|
|
set[e] = true
|
|
}
|
|
}
|
|
|
|
// readWhole says a context is read whole by an artifact.
|
|
func (s *sourceInputs) readWhole(c catalogue.ArtifactContext) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
key := contextKey(c)
|
|
s.readAs[key] = c
|
|
s.read[key] = nil
|
|
}
|
|
|
|
// buildSources is what the build says it was made from, per repository: its own (module.json always,
|
|
// and every artifact's) unless an artifact's is not known, and each context not read whole.
|
|
func (s *sourceInputs) buildSources() []BuildSource {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
var out []BuildSource
|
|
if !s.ownIsAll {
|
|
own := []string{withPrefix(s.prefix, ManifestName)}
|
|
for e := range s.own {
|
|
own = append(own, withPrefix(s.prefix, e))
|
|
}
|
|
sort.Strings(own)
|
|
out = append(out, BuildSource{Paths: compactSorted(own)})
|
|
}
|
|
var keys []string
|
|
for k := range s.read {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
set := s.read[k]
|
|
if set == nil {
|
|
continue
|
|
}
|
|
var paths []string
|
|
for e := range set {
|
|
paths = append(paths, e)
|
|
}
|
|
sort.Strings(paths)
|
|
c := s.readAs[k]
|
|
out = append(out, BuildSource{Repository: c.Repository, Ref: c.Ref, Paths: paths})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// withPrefix is an entry relative to the module's directory made relative to its repository's root.
|
|
func withPrefix(prefix, entry string) string {
|
|
switch {
|
|
case entry == "." || entry == "./":
|
|
entry = ""
|
|
case entry == "**" || entry == "./**" || entry == "/**":
|
|
if prefix == "" {
|
|
return "**"
|
|
}
|
|
return prefix + "/**"
|
|
}
|
|
entry = strings.TrimPrefix(entry, "./")
|
|
if prefix == "" {
|
|
if entry == "" {
|
|
return "./"
|
|
}
|
|
return entry
|
|
}
|
|
if entry == "" {
|
|
return prefix + "/"
|
|
}
|
|
return prefix + "/" + entry
|
|
}
|
|
|
|
func compactSorted(in []string) []string {
|
|
var out []string
|
|
for i, e := range in {
|
|
if i == 0 || e != in[i-1] {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// notPinned marks the build as one its source does not pin; the first reason stands.
|
|
func (s *sourceInputs) notPinned(why string) {
|
|
if s != nil && s.unpinned == "" {
|
|
s.unpinned = why
|
|
}
|
|
}
|
|
|
|
// fingerprint is the build's source fingerprint, or empty when the source does not pin the build.
|
|
func (s *sourceInputs) fingerprint() string {
|
|
if s == nil || s.unpinned != "" || s.tree == "" {
|
|
return ""
|
|
}
|
|
var lines []string
|
|
lines = append(lines, "module "+s.module, "tree "+s.tree)
|
|
bases := map[string]bool{}
|
|
for _, b := range s.bases {
|
|
bases[referenceDigest(b)] = true
|
|
}
|
|
for b := range bases {
|
|
lines = append(lines, "base "+b)
|
|
}
|
|
for a, t := range s.contexts {
|
|
lines = append(lines, "context "+a+" "+t)
|
|
}
|
|
for a, t := range s.toolchains {
|
|
lines = append(lines, "toolchain "+a+" "+t)
|
|
}
|
|
// The module and its tree first, the rest in a fixed order.
|
|
sort.Strings(lines[2:])
|
|
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
|
return sourcePrefix + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// referenceDigest is a reference's digest — `sha256:…` — so the registry address it was copied into does not
|
|
// enter the fingerprint; the reference itself when it carries none.
|
|
func referenceDigest(reference string) string {
|
|
if _, digest, pinned := strings.Cut(reference, "@"); pinned && digest != "" {
|
|
return digest
|
|
}
|
|
return reference
|
|
}
|
|
|
|
// gitTree is the git tree of a directory of a clone at its checked-out commit: the whole tree for an
|
|
// empty path.
|
|
func gitTree(ctx context.Context, run Runner, clone, path string) (string, error) {
|
|
spec := "HEAD^{tree}"
|
|
if rel := strings.Trim(filepath.ToSlash(filepath.Clean(path)), "/"); path != "" && rel != "" && rel != "." {
|
|
spec = "HEAD:" + rel
|
|
}
|
|
out, err := run(ctx, clone, "git", "rev-parse", spec)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
tree := strings.TrimSpace(out)
|
|
if tree == "" {
|
|
return "", fmt.Errorf("git named no tree for %s", spec)
|
|
}
|
|
return tree, nil
|
|
}
|
|
|
|
// toolchainOf is what a bundle's compile adds to its fingerprint: the compiler image by digest and
|
|
// the builder's recipe for the language, hashed, so a builder that compiles differently is a change.
|
|
func toolchainOf(chain Toolchain, base string) string {
|
|
recipe, _ := json.Marshal(chain)
|
|
sum := sha256.Sum256(recipe)
|
|
return chain.Language + " " + referenceDigest(base) + " " + hex.EncodeToString(sum[:8])
|
|
}
|