novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Merge pull request 'A rebuild of an unchanged source is no move, whatever image digest it made (hq issue 280)' (#99) from fix/an-unchanged-source-is-no-move into main

Browse Source
This commit was merged in pull request #99.
This commit is contained in:
mesh-admin
2026-10-06 20:12:06 +00:00
parent b9e0cd34c3 792352dfad
commit 9b6b0c5686
15 changed files with 773 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-builder/main.go
+1
View File
@@ -292,6 +292,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
})
}
result.Against = built.Against
result.SourceFingerprint = built.Source
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
}
cmd/mesh-builder/once.go
+2
View File
@@ -105,6 +105,7 @@ func buildOnce(ctx context.Context, args []string) error {
Ref: *ref,
Manifest: built.Manifest,
Against: built.Against,
Source: built.Source,
}
for _, r := range built.Read {
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
@@ -135,6 +136,7 @@ type onceResult struct {
Made []madeArtifact `json:"made"`
Against []string `json:"against,omitempty"`
Read []readRepository `json:"read,omitempty"`
Source string `json:"source-fingerprint,omitempty"`
}
// readRepository is a repository this build read source from besides the module's own.
cmd/mesh-controller/build.go
+19
View File
@@ -148,6 +148,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
// rebuild the graph rather than a list of names.
Path: result.Path,
// What it was made from (novox/hq issue 280): two builds with one are one build.
SourceFingerprint: result.SourceFingerprint,
}
// When it was asked, which is what orders it against another build of the same module
// (novox/hq 04-ISSUES/219) — not when it was heard.
@@ -580,6 +582,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
short(result.Commit))
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
// what the build the mesh stands on was made from registers that build's artifacts at the new
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
// byte-reproducible, and the bus rebuilt for another module's merge demanded a planned upgrade.
if kept.SourceFingerprint != "" {
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
if err != nil {
return manifest, kept, err
}
if stands != "" && stands != kept.ID && len(raw) > 0 {
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
manifest.Module, short(result.Commit), stands)
manifest = same
}
}
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
if errors.Is(err, inventory.ErrSuperseded) {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
cmd/mesh-controller/bus_step.go
+14 -3
View File
@@ -54,13 +54,15 @@ type busPending struct {
machines []string
from map[string]string
to string
// same are the commits whose build made the same artifacts and manifest as the build the mesh holds.
// same are the commits whose build was made from the same source as the build the mesh holds, or
// made the same artifacts and manifest (novox/hq issue 280).
same map[string]bool
}
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
// two builds made the same artifacts from the same manifest — a rebuild of the same source for another
// module's merge changes nothing the machine runs.
// two builds were made from the same source, or made the same artifacts from the same manifest — a
// rebuild of the same source for another module's merge changes nothing the machine runs, whatever
// image digest it made (novox/hq issue 280).
func (b busPending) moves(machine string) bool {
from, known := b.from[machine]
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
@@ -100,6 +102,15 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro
for commit, refs := range made {
b.same[commit] = refs != "" && refs == made[b.to]
}
sources, err := inv.BuildSourceFingerprints(ctx, b.module)
if err != nil {
return b, err
}
for commit, src := range sources {
if src != "" && src == sources[b.to] {
b.same[commit] = true
}
}
for _, n := range b.machines {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
cmd/mesh-controller/release.go
+48 -4
View File
@@ -72,8 +72,10 @@ var errWalkedElsewhere = errors.New("a plan already walking a build there sends
type moveFacts struct {
current map[string]inventory.CurrentBuild
fps map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
// srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280).
srcs map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
}
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
@@ -85,6 +87,9 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return f, err
}
if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
return f, err
}
if f.passed, err = inv.PassedCommits(ctx); err != nil {
return f, err
}
@@ -92,12 +97,17 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
return f, err
}
// identical is whether two builds of a module put the same thing on a machine: the same commit, or
// builds that made the same artifacts from the same manifest.
// identical is whether two builds of a module put the same thing on a machine: the same commit,
// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its
// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same
// artifacts from the same manifest.
func (f moveFacts) identical(module, a, b string) bool {
if a == b || sameCommit(a, b) {
return true
}
if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] {
return true
}
fa := f.fps[module][a]
return fa != "" && fa == f.fps[module][b]
}
@@ -112,6 +122,40 @@ func (f moveFacts) gated(module, commit string) bool {
return false
}
// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every
// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine
// runs it, when what one was sent is not known, or when the build stands for itself.
func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) {
if build == "" || len(running) == 0 {
return false, nil
}
same, err := inv.SameSourceCommits(ctx, module, build)
if err != nil || len(same) == 0 {
return false, err
}
for _, n := range running {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
return false, err
}
was, carried := sent[module]
if !known || !carried || !inRun(same, was) {
return false, nil
}
}
return true, nil
}
// inRun is whether a commit is one of a run's, however either is abbreviated.
func inRun(run map[string]bool, commit string) bool {
for c := range run {
if sameCommit(c, commit) {
return true
}
}
return false
}
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
cmd/mesh-controller/release_plan.go
+14
View File
@@ -620,6 +620,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
if err != nil {
return false, err
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
// of the build every machine running it was sent, it is registered with that build's artifacts —
// nothing to send, and nothing for a gate to judge.
if state.FirstAt == nil {
if same, err := unchangedOnEvery(ctx, inv, m, state.Build, running); err != nil {
return false, err
} else if same {
now := time.Now().UTC()
state.SentAt = &now
state.Why = "no move: made from the source every machine running it runs"
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
continue
}
}
policy, err := inv.UpgradeOf(ctx, m)
if err != nil {
return false, err
cmd/mesh-controller/same_source_test.go
+209
View File
@@ -0,0 +1,209 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A build whose source is unchanged is never a move (novox/hq issue 280): an image is not
// byte-reproducible, so two builds of one source make two digests, and the rule that a rebuild with
// identical artifacts is no move (ADR 0236) never held for one.
// anImageBuild is a build outcome of an image module: its manifest names the image by the digest made.
func anImageBuild(t *testing.T, module, id, commit, digest, source string, asked time.Time) link.BuildResult {
t.Helper()
image := "registry.invalid:5000/" + module + "/server@sha256:" + digest
manifest, _ := json.Marshal(map[string]any{"module": module, "version": "1",
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": module, "image": image}}})
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: manifest, SourceFingerprint: source,
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: image}},
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
// shelfNames is whether the module the mesh holds names this digest.
func shelfNames(t *testing.T, inv *inventory.Inventory, module, digest string) bool {
t.Helper()
shelf, err := inv.Catalogue(t.Context())
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(shelf[module])
return strings.Contains(string(raw), digest)
}
// A rebuild of an unchanged source is registered with the artifacts of the build it was first made
// as: no machine is sent a new digest, a module standing on it is handed the same base, and the two
// builds are one to every rule that asks whether a send moves something. A real source change moves.
func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
for i, b := range []link.BuildResult{
anImageBuild(t, "app", "", "c1aaaaaa", strings.Repeat("a", 64), "src1:same", start),
// Another module's merge rebuilt it: a new commit, a new image digest, the same source.
anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)),
} {
if _, _, err := takeIn(ctx, inv, b); err != nil {
t.Fatalf("build %d: %v", i, err)
}
}
if !shelfNames(t, inv, "app", strings.Repeat("a", 64)) || shelfNames(t, inv, "app", strings.Repeat("b", 64)) {
t.Fatal("a rebuild of an unchanged source registered the digest it made, not the one the mesh holds")
}
if src, err := inv.SourceOf(ctx, "app"); err != nil || src.BuiltFrom != "c2bbbbbb" {
t.Fatalf("the module is not at the commit it was rebuilt from: %+v %v", src, err)
}
held, err := inv.Held(ctx)
if err != nil || !strings.HasSuffix(held["app/server"], strings.Repeat("a", 64)) {
t.Fatalf("a module standing on it is handed %q, not the build the mesh holds (%v)", held["app/server"], err)
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
if !f.identical("app", "c1aaaaaa", "c2bbbbbb") {
t.Fatal("two builds of one source are not one build")
}
// A real change to the source moves: its own digest registered, and not identical.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c3cccccc", strings.Repeat("c", 64), "src1:changed",
start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("c", 64)) {
t.Fatal("a changed source did not register what it made")
}
if f, _ = readMoveFacts(ctx, inv); f.identical("app", "c2bbbbbb", "c3cccccc") {
t.Fatal("a changed source is read as the same build")
}
// And a builder that says no fingerprint is told apart by its artifacts alone, as before.
if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c4dddddd", strings.Repeat("d", 64), "",
start.Add(3*time.Minute))); err != nil {
t.Fatal(err)
}
if !shelfNames(t, inv, "app", strings.Repeat("d", 64)) {
t.Fatal("a build with no fingerprint did not register what it made")
}
}
// The bus rebuilt for another module's merge, its source untouched and its image digest new: no move —
// no push to its machine is held, no bus step is demanded, and `bus upgrade` has nothing to do. A real
// change to the bus's source is the planned step again.
func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
bus := func(id, commit, digest, source string, at time.Time) link.BuildResult {
b := anImageBuild(t, "nats", id, commit, digest, source, at)
manifest, _ := json.Marshal(map[string]any{"module": "nats", "version": "2",
"provides": []any{map[string]any{"name": "mesh-bus"}},
"resources": []any{map[string]any{"id": "svc", "type": "container", "name": "nats",
"image": b.Made[0].Reference}}})
b.Manifest = manifest
return b
}
if _, _, err := takeIn(ctx, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start)); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1111111"}); err != nil {
t.Fatal(err)
}
// The wide rebuild: a new commit, a new image digest, the same source.
if _, _, err := takeIn(ctx, inv, bus("", "n2222222", strings.Repeat("b", 64), "src1:bus", start.Add(time.Minute))); err != nil {
t.Fatal(err)
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a bus rebuilt from an unchanged source held its machine: %v %v", held, err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) {
return "", errors.New("no snapshot is taken for a bus step nobody needs")
}
var sent [][]string
wasSend := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
if err := busCommand(ctx, []string{"upgrade", "--why", "nothing changed", "--reversible"}); err != nil || len(sent) != 0 {
t.Fatalf("a bus step ran for a bus whose source is unchanged: %v, sent %v", err, sent)
}
if !shelfNames(t, inv, "nats", strings.Repeat("a", 64)) {
t.Fatal("the bus the mesh holds is not the image its machine runs")
}
// A real change to the bus's source: the planned step.
if _, _, err := takeIn(ctx, inv, bus("", "n3333333", strings.Repeat("c", 64), "src1:bus-2.12", start.Add(2*time.Minute))); err != nil {
t.Fatal(err)
}
held, err := busHeld(ctx, inv, []string{"anchor"})
if err != nil || !strings.Contains(held["anchor"], "planned step") {
t.Fatalf("a changed bus did not demand its planned step: %v %v", held, err)
}
}
// A plan's build made from the source every machine running the module runs is not sent and not
// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before.
func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) {
for _, tc := range []struct {
name string
source string
sent [][]string
}{
{"unchanged", "src1:app", nil},
{"changed", "src1:app-changed", [][]string{{"anchor"}}},
} {
t.Run(tc.name, func(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// What anchor and laptop run (c1) was made from src1:app; the plan's build of c2 from the source
// the case says.
for _, b := range []inventory.Build{
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: tc.source,
Asked: time.Now(), At: time.Now()},
} {
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
p := g.plan(t)
p.Modules["app"].Build = "build-2s"
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if !reflect.DeepEqual(g.sent, tc.sent) {
t.Fatalf("sent %v, want %v", g.sent, tc.sent)
}
p = g.plan(t)
s := p.Modules["app"]
if tc.sent == nil && (s.SentAt == nil || s.Gate != nil || !strings.Contains(s.Why, "no move")) {
t.Fatalf("an unchanged source was not read as no move: %+v", s)
}
if tc.sent != nil && (s.Gate == nil || len(s.First) == 0) {
t.Fatalf("a changed source was not sent to its first machine, gated: %+v", s)
}
})
}
}
internal/builder/builder.go
+43 -3
View File
@@ -67,6 +67,12 @@ type Result struct {
// mesh keeps carries no build section, so nothing else could say that a merge there is a
// change to this module (novox/hq 04-ISSUES/131).
Read []catalogue.ArtifactContext
// Source is the build's source fingerprint (source.go): what it was made from — the module's tree,
// the contexts' trees, the bases and toolchains by digest — hashed. Empty where the source does not
// pin the build. Two builds with one fingerprint are one build, whatever digests they made
// (novox/hq issue 280).
Source string
}
// GitCredential is the forge credential a clone may present when the server asks for one.
@@ -161,6 +167,12 @@ func Build(ctx context.Context, run Runner, publish Publisher,
}
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
// What it is made from, for its source fingerprint: the module's own tree first.
src := newSourceInputs(manifest.Module)
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
src.notPinned("its tree could not be named: " + err.Error())
}
// A build-time credential, written into the build context as .npmrc, but ONLY for a module that
// asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc.
// Writing it into every context would put a per-run credential in `COPY . .` of modules that
@@ -178,6 +190,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
}
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
src.notPinned("it resolves packages from the mesh's registry at build time")
}
var built []catalogue.Built
@@ -206,6 +219,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
return Result{}, err
}
stoodOn = bases
src.bases = append(src.bases, bases...)
if len(args) > 0 {
say("bases", "%d resolved from what the mesh holds", len(args)/2)
}
@@ -215,7 +229,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
for _, a := range artifacts {
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
if err != nil {
say("artifact", "%s FAILED: %v", a.Name, err)
return Result{}, err
@@ -231,8 +245,20 @@ func Build(ctx context.Context, run Runner, publish Publisher,
return Result{}, err
}
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
fingerprint := src.fingerprint()
if fingerprint == "" {
say("source", "no source fingerprint: %s", orNoTree(src.unpinned))
}
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest, stoodOn), Read: readBy(manifest)}, nil
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint}, nil
}
// orNoTree is why a build has no source fingerprint, for its log.
func orNoTree(why string) string {
if why == "" {
return "its tree was not named"
}
return why
}
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
@@ -443,7 +469,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
func one(ctx context.Context, run Runner, publish Publisher,
module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
held map[string]string, npmrc string, registry Npmrc, seats map[string]string, src *sourceInputs,
say func(step, format string, args ...any)) (catalogue.Built, error) {
switch a.Kind {
@@ -525,6 +551,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
}
if t, err := gitTree(ctx, run, cloned, ""); err != nil {
src.notPinned(a.Name + "'s context could not be named: " + err.Error())
} else if src != nil {
src.contexts[a.Name] = t
}
// docker build accepts -f outside the context it is given; the recipe stays exactly
// where it was read from and validated against, absolute so the working directory
// switching to the cloned context does not change which file that is.
@@ -582,6 +613,14 @@ func one(ctx context.Context, run Runner, publish Publisher,
"holds no copy of it. Build %s first",
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
}
if src != nil {
src.toolchains[a.Name] = toolchainOf(chain, base)
}
if chain.Language == "typescript" {
if own, _ := ownDependencies(tree); len(own) > 0 {
src.notPinned(a.Name + " resolves packages of its own at build time")
}
}
// The module's own packages first, where the compiler and the bundler resolve them from
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
@@ -623,6 +662,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
// there is no Publisher call — the container itself publishes, with the credential the
// build was handed.
say("package", "building and publishing %s (%s)", a.Name, a.Language)
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
internal/builder/builder_test.go
+4
View File
@@ -36,6 +36,8 @@ type recorded struct {
// carried timestamps would still produce one digest — which is a test that passes for a
// reason that has nothing to do with what it claims.
stamped time.Time
// tree is what git names as the tree of the module's directory; empty answers as for a commit.
tree string
}
func (r *recorded) run(_ context.Context, dir, name string, args ...string) (string, error) {
@@ -77,6 +79,8 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
}
}
return "", nil
case name == "git" && len(args) > 1 && args[0] == "rev-parse" && strings.HasPrefix(args[1], "HEAD:") && r.tree != "":
return r.tree + "\n", nil
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
}
internal/builder/source.go
+124
View File
@@ -0,0 +1,124 @@
package builder
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path/filepath"
"sort"
"strings"
)
// A build's source fingerprint: what it was made from, hashed (novox/hq issue 280).
//
// **An image is not byte-reproducible.** Two builds of one source make two image digests, so the rule
// "a rebuild that made the same artifacts is no move" (novox/hq ADR 0236) held for archives and bundles
// and never for an image: a merge that rebuilt the bus without touching it made a "new" bus build, and
// every send to the machine running it was refused until a planned bus upgrade — for a bus nothing had
// changed. What a build is made from is reproducible, so that is what is fingerprinted:
//
// - the git tree of the module's directory at the commit built — every file the build reads, its
// module.json and its recipes among them, since the recipe and the compiler see that directory only;
// - the git tree of each other repository an artifact's context is cloned from (ArtifactContext);
// - each base it was handed, by digest — a module's artifact or a declared vendor image (build.on);
// - for a bundle, the toolchain it was compiled in: the compiler image's digest and the builder's own
// recipe for that language.
//
// **No fingerprint where the source does not pin the build.** A build that resolves packages from the
// mesh's package registry at build time — a `package` artifact, a TypeScript bundle with packages of
// its own, an image whose recipe reads the registry credential — takes whatever the registry holds
// then, so the same source can be a different program; it records none, and only its artifacts can
// say it is the same. A recipe that fetches from the internet without a pin is the recipe's choice
// (novox/hq ADR 0097 refuses the unpinned bases; what a RUN step downloads is not seen here).
// sourcePrefix names the fingerprint's form, so a later form is never compared equal to this one.
const sourcePrefix = "src1:"
// sourceInputs collects what one build was made from.
type sourceInputs struct {
module string
// tree is the git tree of the module's directory at the commit built.
tree string
// bases are the digests of what the build was handed to stand on.
bases []string
// contexts are, per artifact, the git tree of the repository its context was cloned from.
contexts map[string]string
// toolchains are, per bundle artifact, the compiler image's digest and the recipe's hash.
toolchains map[string]string
// unpinned is why this build has no fingerprint: empty when it has one.
unpinned string
}
func newSourceInputs(module string) *sourceInputs {
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}}
}
// notPinned marks the build as one its source does not pin; the first reason stands.
func (s *sourceInputs) notPinned(why string) {
if s != nil && s.unpinned == "" {
s.unpinned = why
}
}
// fingerprint is the build's source fingerprint, or empty when the source does not pin the build.
func (s *sourceInputs) fingerprint() string {
if s == nil || s.unpinned != "" || s.tree == "" {
return ""
}
var lines []string
lines = append(lines, "module "+s.module, "tree "+s.tree)
bases := map[string]bool{}
for _, b := range s.bases {
bases[referenceDigest(b)] = true
}
for b := range bases {
lines = append(lines, "base "+b)
}
for a, t := range s.contexts {
lines = append(lines, "context "+a+" "+t)
}
for a, t := range s.toolchains {
lines = append(lines, "toolchain "+a+" "+t)
}
// The module and its tree first, the rest in a fixed order.
sort.Strings(lines[2:])
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return sourcePrefix + hex.EncodeToString(sum[:])
}
// referenceDigest is a reference's digest — `sha256:…` — so the registry address it was copied into does not
// enter the fingerprint; the reference itself when it carries none.
func referenceDigest(reference string) string {
if _, digest, pinned := strings.Cut(reference, "@"); pinned && digest != "" {
return digest
}
return reference
}
// gitTree is the git tree of a directory of a clone at its checked-out commit: the whole tree for an
// empty path.
func gitTree(ctx context.Context, run Runner, clone, path string) (string, error) {
spec := "HEAD^{tree}"
if rel := strings.Trim(filepath.ToSlash(filepath.Clean(path)), "/"); path != "" && rel != "" && rel != "." {
spec = "HEAD:" + rel
}
out, err := run(ctx, clone, "git", "rev-parse", spec)
if err != nil {
return "", err
}
tree := strings.TrimSpace(out)
if tree == "" {
return "", fmt.Errorf("git named no tree for %s", spec)
}
return tree, nil
}
// toolchainOf is what a bundle's compile adds to its fingerprint: the compiler image by digest and
// the builder's recipe for the language, hashed, so a builder that compiles differently is a change.
func toolchainOf(chain Toolchain, base string) string {
recipe, _ := json.Marshal(chain)
sum := sha256.Sum256(recipe)
return chain.Language + " " + referenceDigest(base) + " " + hex.EncodeToString(sum[:8])
}
internal/builder/source_test.go
+76
View File
@@ -0,0 +1,76 @@
package builder
import (
"context"
"strings"
"testing"
)
// A build's source fingerprint (novox/hq issue 280): what it was made from, so a rebuild of an unchanged
// source is one build however the image digest it made differs.
const anImage = `{"module":"bus","version":"1",
"build":{"on":[{"arg":"BASE","image":"vendor/server@sha256:` + "1111111111111111111111111111111111111111111111111111111111111111" + `"}],
"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]},
"resources":[{"id":"svc","type":"container","name":"bus","artifact":"server"}]}`
func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string {
t.Helper()
r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
r.contents["modules/bus/module.json"] = manifest
r.tree = tree
m := &mirroring{recorded: r, at: mirrored}
got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
return got.Source
}
// mirroring is a store that copies a vendor's image into the mesh's registry, at an address a test says.
type mirroring struct {
*recorded
at string
}
func (m *mirroring) MirrorImage(_ context.Context, from, repository string) (string, error) {
_, digest, _ := strings.Cut(from, "@")
return m.at + "/" + repository + "@" + digest, nil
}
func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) {
one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000")
if !strings.HasPrefix(one, sourcePrefix) {
t.Fatalf("no fingerprint: %q", one)
}
// The same tree and base, the base copied to another registry address: one source.
if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one {
t.Fatalf("one source has two fingerprints: %s %s", one, again)
}
// The module's tree changed: another source.
if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one {
t.Fatal("a changed tree kept its fingerprint")
}
// The base moved: another source.
moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111",
"2222222222222222222222222222222222222222222222222222222222222222", 1)
if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one {
t.Fatal("a moved base kept its fingerprint")
}
}
func TestABuildTheRegistryDecidesHasNoFingerprint(t *testing.T) {
s := newSourceInputs("app")
s.tree = "aaaa"
if s.fingerprint() == "" {
t.Fatal("a pinned source has no fingerprint")
}
s.notPinned("it resolves packages from the mesh's registry at build time")
if got := s.fingerprint(); got != "" {
t.Fatalf("a build the registry decides has a fingerprint: %s", got)
}
if (&sourceInputs{module: "app"}).fingerprint() != "" {
t.Fatal("a build whose tree was not named has a fingerprint")
}
}
internal/inventory/builds.go
+13 -25
View File
@@ -42,6 +42,9 @@ type Build struct {
// Read is every repository this build read source from besides the module's own (novox/hq
// 04-ISSUES/131), at the ref it read.
Read []ReadRepository
// SourceFingerprint is what the build was made from, hashed, as its builder said it (novox/hq
// issue 280); empty from a builder that predates it, or where the source does not pin the build.
SourceFingerprint string
// Failed is the builder's own words, empty when it worked.
Failed string
Made []Artifact
@@ -112,11 +115,11 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
}
_, err = i.store.Pool().Exec(ctx,
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
source_path, manifest, built_against, built_contexts, asked)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
source_path, manifest, built_against, built_contexts, asked, source_fingerprint)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
on conflict (id) do nothing`,
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
b.Path, manifestOrNil(b.Manifest), against, read, asked)
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint)
return err
}
@@ -202,38 +205,23 @@ func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, erro
// Only successes, and only builds that knew what they were building: a build that failed before it
// could read a manifest has no module to be the artifact of.
func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, made
from build
where module is not null and module <> '' and failed = ''
order by module, `+newestRequestFirst)
// **A rebuild of an unchanged source holds what the first build of it made** (novox/hq issue 280):
// the mesh registers that build's artifacts, so a module standing on it is handed the same base
// and is unchanged too, rather than moving for a digest an image rebuild could not help changing.
made, err := i.heldMade(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
held := map[string]string{}
for rows.Next() {
var module string
var raw []byte
if err := rows.Scan(&module, &raw); err != nil {
return nil, err
}
var made []Artifact
if err := json.Unmarshal(raw, &made); err != nil {
// Skipped rather than fatal. One unreadable build record should not stop every other
// module's base from being answerable — and the build that needs this one will say
// plainly that it is missing.
continue
}
for _, artifact := range made {
for module, artifacts := range made {
for _, artifact := range artifacts {
if artifact.Name == "" || artifact.Reference == "" {
continue
}
held[module+"/"+artifact.Name] = artifact.Reference
}
}
return held, rows.Err()
return held, nil
}
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
internal/inventory/migrations/0074-a-build-says-what-it-was-made-from.sql
+13
View File
@@ -0,0 +1,13 @@
-- A build says what it was made from (novox/hq issue 280).
--
-- A rebuild was "no move" only when it made the same artifacts (novox/hq ADR 0236), and an image is not
-- byte-reproducible: a merge that rebuilt the bus without touching its source made a new bus image
-- digest, the mesh read it as a new bus build, and every send to the machine running the bus was
-- refused until a planned bus upgrade. The builder now says what the build was made from — the git
-- tree of the module's directory, the trees of the contexts it read, its bases and toolchains by
-- digest — hashed, and two builds with one source fingerprint are one build.
--
-- Empty for every build recorded before this and for a build whose source does not pin it (one that
-- resolves packages from the registry at build time): those are told apart by their artifacts alone,
-- exactly as before.
alter table build add column source_fingerprint text not null default '';
internal/inventory/source.go
+186
View File
@@ -0,0 +1,186 @@
package inventory
import (
"context"
"encoding/json"
)
// A build whose source is unchanged is never a move (novox/hq issue 280).
//
// The builder says what each build was made from, hashed (its source fingerprint). Two successful
// builds of a module with one fingerprint are one build, whatever digests they made — an image is not
// byte-reproducible, and reading a rebuild's new image digest as a new build made a merge that never
// touched the bus demand a planned bus upgrade before anything could be sent to the machine running
// it. So:
//
// - a module's builds, newest request first, fall into runs of one fingerprint; **the oldest build of
// the newest run stands for the run**: its artifacts are the ones the mesh registers and hands
// every module that stands on it (Held), so a rebuild of an unchanged source changes nothing any
// machine is sent, and nothing standing on it moves either;
// - a build that failed its gate ends a run: what was put back is never what a later build stands for;
// - an empty fingerprint — a builder that predates it, a source that does not pin its build — is a
// run of its own, as every build was before.
// sourceRow is one successful build of a module, as the runs are read.
type sourceRow struct {
id, commit, fingerprint string
made []byte
manifest []byte
failedGate bool
}
// sourceRows is every successful build of each module (of one module, when named), newest request
// first.
func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) {
query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest,
coalesce(g.verdict = 'failed', false)
from build b left join build_gate g on g.build = b.id
where b.module is not null and b.module <> '' and b.failed = ''`
args := []any{}
if module != "" {
query += ` and b.module = $1`
args = append(args, module)
}
rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...)
if err != nil {
return nil, nil, err
}
defer rows.Close()
out := map[string][]sourceRow{}
var order []string
for rows.Next() {
var m string
var r sourceRow
if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil {
return nil, nil, err
}
if _, seen := out[m]; !seen {
order = append(order, m)
}
out[m] = append(out[m], r)
}
return out, order, rows.Err()
}
// standing is, of a module's builds newest first, the index of the build that stands for the build at
// `from`: the oldest of the unbroken run of builds behind it with its source fingerprint.
func standing(builds []sourceRow, from int) int {
at := from
fp := builds[from].fingerprint
if fp == "" || builds[from].failedGate {
return at
}
for j := from + 1; j < len(builds); j++ {
if builds[j].fingerprint != fp || builds[j].failedGate {
break
}
at = j
}
return at
}
// StandingBuild is the build that stands for a module's build (novox/hq issue 280): the oldest build of
// the unbroken run of builds with its source fingerprint, at or before it — itself when its fingerprint
// is empty or it starts the run. Answers its id and the manifest it was recorded with; empty when the
// build is not a successful build of the module on record.
func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (string, []byte, error) {
all, _, err := i.sourceRows(ctx, module)
if err != nil {
return "", nil, err
}
builds := all[module]
for k, b := range builds {
if b.id == build {
s := builds[standing(builds, k)]
return s.id, s.manifest, nil
}
}
return "", nil, nil
}
// SourceFingerprints is, per module, per commit, the source fingerprint of the newest successful build
// from it that has one: module → commit → fingerprint. A commit whose builds carry none is absent.
func (i *Inventory) SourceFingerprints(ctx context.Context) (map[string]map[string]string, error) {
all, _, err := i.sourceRows(ctx, "")
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for m, builds := range all {
for _, b := range builds {
if b.fingerprint == "" || b.commit == "" {
continue
}
if out[m] == nil {
out[m] = map[string]string{}
}
if _, seen := out[m][b.commit]; !seen {
out[m][b.commit] = b.fingerprint
}
}
}
return out, nil
}
// BuildSourceFingerprints is SourceFingerprints for one module: commit → fingerprint.
func (i *Inventory) BuildSourceFingerprints(ctx context.Context, module string) (map[string]string, error) {
all, err := i.SourceFingerprints(ctx)
if err != nil {
return nil, err
}
if all[module] == nil {
return map[string]string{}, nil
}
return all[module], nil
}
// heldMade is, per module, what the build standing for its newest successful build made — what Held
// answers (novox/hq issue 280).
func (i *Inventory) heldMade(ctx context.Context) (map[string][]Artifact, error) {
all, _, err := i.sourceRows(ctx, "")
if err != nil {
return nil, err
}
out := map[string][]Artifact{}
for m, builds := range all {
if len(builds) == 0 {
continue
}
var made []Artifact
if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err != nil {
// Skipped rather than fatal, as Held always did: the build that needs it says it is missing.
continue
}
out[m] = made
}
return out, nil
}
// SameSourceCommits is the commits of the builds in a build's run (novox/hq issue 280): the build and
// every build behind it back to the one standing for it, all made from one source. Empty when the
// build stands for itself — no earlier build was made from its source — so a commit alone never says
// two builds are one: a dependent rebuilt because its base moved keeps its commit and is a move.
func (i *Inventory) SameSourceCommits(ctx context.Context, module, build string) (map[string]bool, error) {
all, _, err := i.sourceRows(ctx, module)
if err != nil {
return nil, err
}
builds := all[module]
for k, b := range builds {
if b.id != build {
continue
}
s := standing(builds, k)
if s == k {
return nil, nil
}
out := map[string]bool{}
for j := k; j <= s; j++ {
if builds[j].commit != "" {
out[builds[j].commit] = true
}
}
return out, nil
}
return nil, nil
}
internal/link/build.go
+7
View File
@@ -179,6 +179,13 @@ type BuildResult struct {
// manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131).
Read []ReadRepository `json:"read,omitempty"`
// SourceFingerprint is what the build was made from, hashed (novox/hq issue 280): the module's
// tree at the commit, the trees of the contexts it read, its bases and toolchains by digest. Two
// builds with one fingerprint are one build, however their digests differ — an image is not
// byte-reproducible. Empty from a builder that predates it, or where the source does not pin the
// build; then only identical artifacts make a rebuild no move.
SourceFingerprint string `json:"source-fingerprint,omitempty"`
// Failed is why, when it did.
Failed string `json:"failed,omitempty"`