novox/mesh-controller
7
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

One build per change: the check builds what its merge moves, the walk takes it by key (ADR 0350, issue 649) #311

Merged
mesh-admin merged 2 commits from feat/350-one-build-per-change into main 2026-10-13 21:30:25 +00:00
Conversation 3 Commits 2 Files Changed 25
+2195 -13
23 changed files with 2195 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit cd49b50c40 - Show all commits
cmd/mesh-builder/holder.go
+31
View File
@@ -41,6 +41,9 @@ type holder struct {
say func(link.HolderState) error
// remove runs what a kill needs outside the build: the containers left behind.
remove builder.Runner
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
// this machine publishes to no artifact store.
scratch builder.Scratch
mu sync.Mutex
paused bool
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
}
}
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
// after. Every holder sweeps the one store; what one let go of, another finds gone.
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
sweep := func() {
if h.scratch == nil {
return
}
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
switch {
case err != nil:
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
case n > 0:
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
}
}
sweep()
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
sweep()
}
}
}
// begin records the build this machine took, with the cancel that ends it.
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
cmd/mesh-builder/main.go
+62 -5
View File
@@ -97,6 +97,9 @@ func run() error {
if err := notReady(); err != nil {
return err
}
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
// differently never takes a build another one made.
builder.Version = builder.SelfVersion(version)
credential, err := brokerFrom()
if err != nil {
return err
@@ -174,6 +177,9 @@ func run() error {
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
publisher := builder.Registry{Address: registry, Run: builder.Command}
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
h.scratch = publisher
go h.expireChecks(ctx, time.Hour, time.Now)
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
answer(ctx, publisher, on, workspace, work, h)
@@ -262,6 +268,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if err == nil {
say("packages", saysRegistry(npmrc))
}
registryHere, isRegistry := publisher.(builder.Registry)
var built builder.Result
if request.Check != nil {
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
@@ -271,12 +278,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if r, ok := publisher.(builder.Registry); ok {
registry = r.Address
}
spec := checkSpecOf(request)
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
if isRegistry && len(spec.Builds) > 0 {
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
Seats: request.Seats}
}
var v builder.CheckVerdict
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)}
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
for _, m := range v.Members {
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
@@ -289,9 +305,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
// after a clone that then fails at npm ci.
// Every container it starts is labelled with its id, so a kill finds what outlived the
// docker client (ADR 0219).
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
if request.Reuse != nil && isRegistry && !request.DryRun {
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
reuse := builder.Reuse{Scratch: registryHere}
for _, p := range request.Reuse.Pulls {
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
}
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
} else {
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
}
}
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
result.Mirrored = built.Mirrored
@@ -327,6 +354,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
}
result.Against = built.Against
result.SourceFingerprint = built.Source
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
result.ReuseKey = built.ReuseKey
if built.Reused != nil {
f := built.Reused.From
result.Reused = built.Reused.Words()
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
} else if request.Reuse != nil {
result.BuiltWhy = built.Why
if result.BuiltWhy == "" {
result.BuiltWhy = "it has no artifact to build"
}
}
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
@@ -377,6 +417,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
for dir, b := range c.Beside {
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
}
for _, b := range c.Builds {
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
}
for _, m := range c.Members {
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
@@ -384,6 +428,19 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
return spec
}
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
var out []link.CheckBuilt
for _, b := range built {
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()}
for _, m := range b.Made {
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
}
out = append(out, c)
}
return out
}
// layerOf is one layer of a check as the outcome carries it.
func layerOf(l *builder.Layer) *link.CheckLayer {
if l == nil {
cmd/mesh-controller/build.go
+18
View File
@@ -15,6 +15,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -483,6 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
Seats: seatBases(ctx),
DryRun: dryRun,
}
request.Reuse = reuseAsked(ctx, dryRun)
fmt.Printf("asked for %s", source)
if source.Seat != "" {
fmt.Printf(" (%s)", repository)
@@ -669,6 +671,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
manifest.Module, short(result.Commit), err)
}
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
// registered only as the walk's copy in the module's own namespace.
if scratch := inScratch(result.Made); scratch != "" {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
}
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
// and the next push would send it.
@@ -714,6 +722,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, nil
}
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
func inScratch(made []link.MadeArtifact) string {
for _, m := range made {
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
return m.Reference
}
}
return ""
}
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
"never published (ADR 0238) — merge it, and the merge builds it")
cmd/mesh-controller/check_builds.go
+87
View File
@@ -0,0 +1,87 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"strconv"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/inventory"
)
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
// so a word lost here leaves a week's builds, never more.
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
address, err := artifactStoreAddress(ctx, inv, shelf, "")
if err != nil {
return nil, err
}
if address == "" {
return nil, errors.New("the artifact store is not on the mesh's network")
}
return builder.Registry{Address: address}, nil
}
// forgetChecks lets go of a pull request's check builds and answers what it did.
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
scratch, err := checkBuildsIn(ctx, inv)
if err != nil {
return "", err
}
n, err := scratch.Forget(ctx, pull)
if err != nil {
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
}
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
}
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
// it answers: the walk took what it reuses. Said, never fatal.
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
return
}
for _, pull := range reuseAskOf(&p).Pulls {
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
if err != nil {
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
continue
}
fmt.Printf("%s: %s\n", p.ID, said)
}
}
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
func forgetChecksCommand(ctx context.Context, args []string) error {
if len(args) != 3 {
return errors.New("forget-checks <owner> <repo> <number>")
}
number, err := strconv.Atoi(args[2])
if err != nil || number <= 0 {
return fmt.Errorf("%q is not a pull request's number", args[2])
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
if err != nil {
return err
}
fmt.Println(said)
return nil
}
cmd/mesh-controller/check_builds_test.go
+190
View File
@@ -0,0 +1,190 @@
package main
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
// that reads the repository through a context not at all. A change that moves no module builds nothing.
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
entries := []inventory.Entry{
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
}
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
got := checkBuildsOf(p, scope, entries, clone)
want := []link.CheckBuild{
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
}
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
t.Fatalf("a change that moves no module asks for builds: %+v", got)
}
}
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
// run never does, and neither does an ask that is not a walk's.
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
asked := reuseAskOf(p)
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
t.Fatalf("the walk names %+v", asked.Pulls)
}
walk := withReuse(t.Context(), asked)
if reuseAsked(walk, false) != asked {
t.Fatal("a walk's ask does not carry its leave")
}
if reuseAsked(walk, true) != nil {
t.Fatal("a dry run carries a leave to take a check's build")
}
if reuseAsked(t.Context(), false) != nil {
t.Fatal("an ask that is not a walk's carries a leave")
}
}
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
var forgot []builder.PullRef
was := forgetChecks
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
forgot = append(forgot, pull)
return "", nil
}
t.Cleanup(func() { forgetChecks = was })
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
{Repository: "novox/shop", Number: 7}}}}
p.WalkSaid = inventory.WalkStarted
forgetChecksOfWalk(t.Context(), nil, p)
p.WalkSaid = ""
forgetChecksOfWalk(t.Context(), nil, p)
if len(forgot) != 0 {
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
}
p.WalkSaid = inventory.WalkEnded
forgetChecksOfWalk(t.Context(), nil, p)
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
t.Fatalf("the walk ended let go of %v", forgot)
}
}
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
t.Fatalf("a module's own build reads as a scratch build: %s", got)
}
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
t.Fatalf("a scratch build was not found: %q", got)
}
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
t.Fatal("a scratch archive was not found")
}
}
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
// neither, which is said.
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
t.Fatalf("said %q", got)
}
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
t.Fatalf("said %q", got)
}
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
t.Fatalf("said %q", got)
}
}
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
// is named in testdata/store-tests.
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
gateEvery = 0
digest := "sha256:" + strings.Repeat("c", 64)
reused := func(id, reference string) link.BuildResult {
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
}
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
!strings.Contains(err.Error(), "scratch") {
t.Fatalf("a build in the scratch namespace was registered: %v", err)
}
id := link.NewBuildID(time.Now().Add(time.Second))
result := reused(id, "store.internal:5000/app/server@"+digest)
if _, _, err := takeIn(ctx, inv, result); err != nil {
t.Fatal(err)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil || current["app"].Commit != "c3" {
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(shelf["app"].Resources)
if !strings.Contains(string(raw), "app/server@"+digest) {
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
}
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
asked := time.Now().UTC()
p := g.plan(t)
p.Commit = "c3"
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
p = g.plan(t)
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
t.Fatalf("the walk keeps %+v", m)
}
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p = g.plan(t)
gate := p.Modules["app"].Gate
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
t.Fatalf("the gate judged %+v, not the reused build", gate)
}
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
t.Fatalf("the first machine was sent %v, not the reused build", sent)
}
}
cmd/mesh-controller/checks.go
+51 -1
View File
@@ -282,10 +282,59 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
Manifests: scope.Manifests, Judge: scope.Judge},
Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)},
}, nil
}
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
// moves no module builds nothing.
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
clone func(inventory.Source) (string, error)) []link.CheckBuild {
if len(scope.Reach.Plan.Tiers) == 0 {
return nil
}
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
skip := map[string]string{}
for _, e := range scope.Reach.Packaging {
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
"cannot hand the merge"
}
for _, e := range scope.Reach.Deleted {
skip[e.Manifest.Module] = "the merge deletes it"
}
var out []link.CheckBuild
for tier, names := range scope.Reach.Plan.Tiers {
for _, name := range names {
e, known := byName[name]
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
switch {
case !known:
b.Skip = "the catalogue does not hold it"
case b.Skip != "":
case sourceIs(e.Source, m):
b.Merged, b.Path = true, e.Source.Path
default:
url, err := clone(e.Source)
if err != nil {
b.Skip = "its repository cannot be cloned: " + err.Error()
break
}
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
}
out = append(out, b)
}
}
return out
}
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
@@ -382,6 +431,7 @@ func checkedOf(result link.BuildResult) link.Checked {
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
// repositories read beside it.
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
c.Built = result.Check.Built
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
cmd/mesh-controller/main.go
+9 -1
View File
@@ -104,6 +104,8 @@ func run() error {
return replayCommand(ctx, args[1:])
case "kill":
return killCommand(ctx, args[1:])
case "forget-checks":
return forgetChecksCommand(ctx, args[1:])
case "pause", "resume":
return pauseCommand(ctx, args[0], args[1:])
case "collection":
@@ -429,8 +431,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
}
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
// before it says nothing, and that is what is kept.
made := result.MadeWords()
if result.Reused != "" || result.BuiltWhy != "" {
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
}
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
// A module registered may be one a machine is now behind: `status` is composed again.
statusFrom.nudge()
return nil
cmd/mesh-controller/push.go
+7 -1
View File
@@ -249,7 +249,13 @@ func serve(ctx context.Context) (err error) {
// (novox/hq to-be 45 §9).
checkEvents = bus
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
sayPlanMoved(ctx, bus, open.inventory, p)
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
if p.WalkSaid == inventory.WalkEnded {
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
}
}
if err := server.Checks(following{open}); err != nil {
return err
}
cmd/mesh-controller/release_plan.go
+64 -2
View File
@@ -313,6 +313,59 @@ func init() {
}
}
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
type reuseKey struct{}
// withReuse is ctx carrying a walk's leave to take a check's build.
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
return context.WithValue(ctx, reuseKey{}, r)
}
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
func reuseIn(ctx context.Context) *link.ReuseAsk {
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
return r
}
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
// takes nothing either (novox/hq ADR 0350 §2, §6).
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
if dryRun {
return nil
}
return reuseIn(ctx)
}
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
r := &link.ReuseAsk{}
if p.Delivery == nil {
return r
}
seen := map[link.PullRef]bool{}
for _, m := range p.Delivery.Merges {
pull, ok := pullOf(m)
if !ok {
continue
}
if !seen[pull] {
seen[pull] = true
r.Pulls = append(r.Pulls, pull)
}
}
return r
}
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/")
if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" {
return link.PullRef{}, false
}
return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true
}
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
@@ -340,7 +393,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
ref = carried
}
id, err := askABuild(ctx, source, e.Source.Path, ref)
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
// requests it answers, whose checks name the part of a differing key.
id, err := askABuild(withReuse(ctx, reuseAskOf(p)), source, e.Source.Path, ref)
if err != nil {
state.State = "failed"
state.Why = err.Error()
@@ -359,7 +414,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
state.State = "asked"
state.AskedAt = &now
state.Build = id
state.Why, state.Commit, state.BuiltAt = "", "", nil
state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, ""
}
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
@@ -371,6 +426,12 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
// build's request time is not known, and such an outcome is taken as before.
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
}
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
// ADR 0350 §4) — on the module of the walk it answers.
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
inv := open.inventory
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
// than write over what the holder is about to save. Not taken, it is still in the build records,
@@ -436,6 +497,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
state.State = "built"
state.BuiltAt = &now
state.Commit = commit
state.Made = made
}
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
cmd/mesh-controller/seatverbs.go
+5
View File
@@ -452,6 +452,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
case "queue":
// As data, the table its output (novox/hq issue 455).
return []string{"queue", "--json"}, nil
case "forget-checks":
if err := need("owner", "repo", "number"); err != nil {
return nil, err
}
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
case "cancel", "kill":
if err := need("id"); err != nil {
return nil, err
internal/artifacts/scratch.go
+91
View File
@@ -0,0 +1,91 @@
package artifacts
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
)
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
// or it is seven days old, so the store keeps a week of check builds and no more.
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
if s.Address == "" {
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
}
var tags []string
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
for next != "" {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
if err != nil {
return nil, err
}
response, err := s.client().Do(request)
if err != nil {
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
response.Body.Close()
return tags, nil
default:
response.Body.Close()
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
}
var page struct {
Tags []string `json:"tags"`
}
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
link := response.Header.Get("Link")
response.Body.Close()
if err != nil {
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
}
tags = append(tags, page.Tags...)
next = nextPage(link, s.Address)
}
return tags, nil
}
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
func nextPage(link, address string) string {
if !strings.Contains(link, `rel="next"`) {
return ""
}
start, end := strings.Index(link, "<"), strings.Index(link, ">")
if start < 0 || end <= start+1 {
return ""
}
target := link[start+1 : end]
if strings.HasPrefix(target, "/") {
return "http://" + address + target
}
return target
}
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
// collection takes its layer. Gone when the tag names nothing.
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
}
digest, err := s.manifestDigest(ctx, repository, tag)
if err != nil {
return err
}
if digest == "" {
return Gone
}
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
if errors.Is(err, Gone) {
return Gone
}
return err
}
internal/builder/builder.go
+184 -3
View File
@@ -91,6 +91,16 @@ type Result struct {
// (novox/hq issue 280).
Source string
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's
// version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build.
ReuseKey string
ReuseParts map[string]string
NoKey string
// Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did
// not take one (§3). Both empty for a build not asked to reuse.
Reused *Reused
Why string
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
@@ -136,14 +146,41 @@ func Build(ctx context.Context, run Runner, publish Publisher,
// one record that it is there.
var mirrored []string
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
&mirrored, seats...)
&mirrored, nil, seats...)
result.Mirrored = mirrored
return result, err
}
// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check
// builds, and the pull requests the walk answers, whose checks name a differing key.
type Reuse struct {
Scratch Scratch
Pulls []PullRef
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
// built at all — nothing could ever take it.
keyFirst bool
}
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
var errNoKey = errors.New("it has no reuse key")
// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse
// key of the commit's tree, and where a check build holds the same key it copies that build into the module's
// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3).
// Only a commit on its trunk takes a check build: anything else builds, as Build does.
func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
var mirrored []string
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
&mirrored, &reuse, seats...)
result.Mirrored = mirrored
return result, err
}
func build(ctx context.Context, run Runner, publish Publisher,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) {
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
// that hand none — tests of everything but contexts — read as they did.
var seatBases map[string]string
@@ -284,6 +321,9 @@ func build(ctx context.Context, run Runner, publish Publisher,
var built []catalogue.Built
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
var stoodOn []string
// reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350).
var reused *Reused
builtWhy := ""
if manifest.Build != nil {
// What this module said it stands on, answered with what this mesh actually holds. Done
// before anything is built, so a missing base is refused in front of the person who can
@@ -327,7 +367,35 @@ func build(ctx context.Context, run Runner, publish Publisher,
// Ordered, so two builds of one commit do the same work in the same sequence and their
// logs can be compared.
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
// **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
if reuse != nil && reuse.keyFirst {
_, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
held, npmrcPath, npmrc, seatBases, src, say)
if err == nil && key.Key == "" {
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
}
if err != nil {
say("reuse", "not built for the merge: %v", err)
return Result{Commit: commit, NoKey: err.Error()}, err
}
} else if reuse != nil && reuse.Scratch != nil {
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
if taken != nil {
built = taken.Made
src = keyed
reused = taken
} else {
builtWhy = why
say("reuse", "built: %s", why)
}
}
for _, a := range artifacts {
if reused != nil {
break
}
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
if err != nil {
@@ -363,9 +431,102 @@ func build(ctx context.Context, run Runner, publish Publisher,
say("source", "%d path(s) of %s", len(s.Paths), where)
}
}
// The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by.
key := keyOf(src, manifest, Version)
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil
}
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into
// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A
// copy that fails is said, and the walk builds (§3): it never fails the walk on its own.
func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest,
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool,
say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) {
if !onTrunk {
return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build"
}
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
held, npmrcPath, npmrc, seats, src, say)
if err != nil {
return nil, nil, err.Error()
}
if key.Key == "" {
return nil, nil, "it has no reuse key: " + key.Why
}
found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key)
if err != nil {
return nil, nil, "the scratch namespace could not be read: " + err.Error()
}
if !ok {
return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key)
}
taken := &Reused{From: found}
for _, made := range found.Made {
copied, err := reuse.Scratch.Take(ctx, manifest.Module, made)
if err != nil {
why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err)
say("reuse", "%s; built instead", why)
return nil, nil, why
}
taken.Made = append(taken.Made, copied)
}
// Every artifact the manifest declares, or none: a check build missing one is not this build.
if len(taken.Made) != len(artifacts) {
return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made),
manifest.Module, len(artifacts))
}
say("reuse", "%s", taken.Words())
return taken, keyed, ""
}
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs,
say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) {
keyed := src.copy()
keyed.onlyKey = true
for _, a := range artifacts {
if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held,
npmrcPath, npmrc, seats, keyed, say); err != nil {
return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err)
}
}
keyed.onlyKey = false
return keyed, keyOf(keyed, manifest, Version), nil
}
// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the
// key differs — naming the part — or the build it made is gone.
func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string {
if len(reuse.Pulls) == 0 {
return "no check built it with this key"
}
builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls)
if err != nil {
return "no check build has this key, and which checks built it could not be read: " + err.Error()
}
for _, b := range builds {
if b.Module != module {
continue
}
if b.Key == key.Key {
return b.Said() + " built it with this key, and it was removed"
}
return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ")
}
var pulls []string
for _, p := range reuse.Pulls {
pulls = append(pulls, p.String())
}
return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " +
"that did not finish)"
}
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
@@ -754,6 +915,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
switch a.Kind {
case catalogue.ArtifactUpstream:
// Its key is the image it copies, read from the manifest (keyOf): nothing to take here.
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
//
@@ -882,6 +1047,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
} else {
src.ownWhole()
}
// Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built.
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
if a.Target != "" {
invocation = append(invocation, "--target", a.Target)
@@ -949,6 +1118,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
src.notPinned(a.Name + " resolves packages of its own at build time")
}
}
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
// The module's own packages first, where the compiler and the bundler resolve them from
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
@@ -992,6 +1164,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
say("package", "building and publishing %s (%s)", a.Name, a.Language)
src.ownWhole()
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
@@ -1000,6 +1175,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
case catalogue.ArtifactArchive:
if src.keyOnly() {
if a.URL == "" {
src.ownHas(cleanEntry(a.From) + "/**")
}
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
var body []byte
var err error
if a.URL != "" {
internal/builder/check.go
+36
View File
@@ -98,6 +98,11 @@ type CheckSpec struct {
// and composed with it by the gate as one future state. The repository's own check is not run for a
// group: each member's pull request runs its own.
Group []GroupHead
// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
// for the walk to take. None for a group's composed check, whose members' own checks build.
Builds []CheckBuildSpec
BuildWith *CheckBuilding
}
// GroupHead is one other head of a delivery group's composed check.
@@ -155,6 +160,8 @@ type CheckVerdict struct {
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
// commit it was merged into (ADR 0323 §6).
Members []JudgedHead
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
Built []CheckBuilt
}
// Layer is one layer of a check, judged.
@@ -572,9 +579,38 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
if why := skipsTheBuilds(v); why != "" {
say("check", "%s", why)
} else {
began := time.Now()
v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
func(step, message string) {
if log != nil {
log(step, redact.redact(message))
}
}, say)
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
time.Since(began).Round(time.Second))
}
}
return named(v), nil
}
// builtCount is how many of a check's builds for its merge were built.
func builtCount(built []CheckBuilt) int {
n := 0
for _, b := range built {
if b.NotBuilt == "" {
n++
}
}
return n
}
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
// answers as the context's error.
internal/builder/check_builds.go
+151
View File
@@ -0,0 +1,151 @@
package builder
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
// it did not fail: a change that fails is not merged, and nothing it built would be taken.
// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
type CheckBuildSpec struct {
Module string
// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
// module is built from the repository checked, at the branch merged into.
Repository string
Path string
Ref string
Merged bool
Tier int
// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
Skip string
}
// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
type CheckBuilding struct {
Publish Publisher
Scratch Scratch
Held map[string]string
Npmrc Npmrc
Seats map[string]string
}
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
// why it was not built.
type CheckBuilt struct {
Module string
Key string
Made []catalogue.Built
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
NotBuilt string
Took time.Duration
}
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
func skipsTheBuilds(v CheckVerdict) string {
for _, l := range []*Layer{v.Gate, v.Repo} {
if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
return "the check did not pass, so nothing is built for its merge"
}
}
return ""
}
// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there.
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
return nil
}
ref := merge
if ref == "" {
ref = spec.Ref
}
held := map[string]string{}
for k, v := range with.Held {
held[k] = v
}
var out []CheckBuilt
var kept []CheckBuild
stopAt, stopWhy := -1, ""
for _, b := range spec.Builds {
began := time.Now()
if stopAt >= 0 && b.Tier > stopAt {
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
continue
}
if b.Skip != "" {
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
}
continue
}
repository, at := b.Repository, b.Ref
if b.Merged {
repository, at = tree, ref
}
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
var mirrored []string
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true},
with.Seats)
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
switch {
case errors.Is(err, errNoKey):
built.NotBuilt = err.Error()
case err != nil:
built.NotBuilt = "its build failed: " + err.Error()
case result.ReuseKey == "":
built.NotBuilt = "it has no reuse key: " + result.NoKey
}
if built.NotBuilt != "" {
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
out = append(out, built)
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
}
continue
}
built.Made = result.Built
for _, m := range result.Built {
held[b.Module+"/"+m.Name] = m.Reference
}
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
out = append(out, built)
if len(result.Built) > 0 {
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
}
}
_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
if len(kept) > 0 {
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
time.Now().UTC(), kept); err != nil {
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
if err := with.Scratch.Discard(ctx, kept); err != nil {
say("build", "and what they made could not all be let go of: %v", err)
}
for i := range out {
if out[i].NotBuilt == "" {
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
}
}
}
}
return out
}
internal/builder/reuse.go
+539
View File
@@ -0,0 +1,539 @@
package builder
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/catalogue"
)
// One build per change (novox/hq ADR 0350).
//
// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
//
// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
// key, and is always built.
// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
// key, every build its own — when it cannot be said.
var Version string
// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
// builder would read as one).
func SelfVersion(stamped string) string {
if exe, err := os.Executable(); err == nil {
if f, err := os.Open(exe); err == nil {
defer f.Close()
sum := sha256.New()
if _, err := io.Copy(sum, f); err == nil {
return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
}
}
}
if stamped == "" || stamped == "development" {
return ""
}
return stamped
}
// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
const ScratchPrefix = "scratch/"
// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
// What the holder lets go of by (§5).
const checksRepository = ScratchPrefix + "checks"
// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
const reuseTagPrefix = "r1-"
// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
const CheckBuildsKept = 7 * 24 * time.Hour
// The parts of a reuse key, as the walk names the one that differs (§3).
const (
partModule = "the module"
partTree = "the build source"
partBases = "a base image"
partBuilder = "the builder's version"
partContext = "a context"
partTool = "a toolchain"
partUpstream = "an upstream image"
)
// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
type reuseKey struct {
Key string
Parts map[string]string
Why string
}
// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
if src == nil || src.unpinned != "" || src.tree == "" {
why := "its source does not pin the build"
if src != nil && src.unpinned != "" {
why = src.unpinned
} else if src != nil && src.tree == "" {
why = "its tree could not be named"
}
return reuseKey{Why: why}
}
if version == "" {
return reuseKey{Why: "the builder's version cannot be said"}
}
parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
var bases []string
seen := map[string]bool{}
for _, b := range src.bases {
if d := referenceDigest(b); !seen[d] {
seen[d] = true
bases = append(bases, d)
}
}
sort.Strings(bases)
parts[partBases] = strings.Join(bases, " ")
for a, t := range src.contexts {
parts[partContext+" "+a] = t
}
for a, t := range src.toolchains {
parts[partTool+" "+a] = t
}
// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
if manifest.Build != nil {
for _, a := range manifest.Build.Artifacts {
if a.Kind != catalogue.ArtifactUpstream {
continue
}
if !strings.Contains(a.From, "@sha256:") {
return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
}
parts[partUpstream+" "+a.Name] = a.From
}
}
lines := make([]string, 0, len(parts))
for k, v := range parts {
lines = append(lines, k+"="+v)
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
}
// tagOf is the tag a key's record is kept under.
func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
func differing(was, now map[string]string) []string {
names := map[string]bool{}
for k := range was {
names[k] = true
}
for k := range now {
names[k] = true
}
var out []string
for k := range names {
if was[k] != now[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
// PullRef names a pull request.
type PullRef struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
}
func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
func (p PullRef) slug() string {
clean := func(s string) string {
return strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
return r
}
return '-'
}, s)
}
return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
}
// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
type CheckBuild struct {
Key string `json:"key"`
Module string `json:"module"`
Parts map[string]string `json:"parts"`
// Made are its artifacts, in the scratch namespace.
Made []catalogue.Built `json:"made"`
// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
// check's ask; At when it was built.
Pull PullRef `json:"pull"`
Head string `json:"head,omitempty"`
Trunk string `json:"trunk,omitempty"`
Check string `json:"check"`
At time.Time `json:"at"`
}
// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
func (c CheckBuild) Said() string {
s := "the check of " + c.Pull.String()
if c.Head != "" {
s += " at head " + twelve(c.Head)
}
if c.Trunk != "" {
s += " (merged into trunk " + twelve(c.Trunk) + ")"
}
return s
}
// checkIndex is one check's document under checksRepository: every build it made.
type checkIndex struct {
Pull PullRef `json:"pull"`
Check string `json:"check"`
At time.Time `json:"at"`
Builds []CheckBuild `json:"builds"`
}
// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
// Registry is one; a test's is a map.
type Scratch interface {
// Keep records a check's builds: each under its module and key, and the check's own document.
Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
// Find is the check build kept under a module's key; false when none is.
Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
// key from.
OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
// each answers how many checks it let go of.
Forget(ctx context.Context, pull PullRef) (int, error)
Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
// Discard lets go of what builds a check could not keep made: nothing would ever find them.
Discard(ctx context.Context, builds []CheckBuild) error
}
// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
type scratchPublisher struct{ to Publisher }
func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
}
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
}
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
m, can := s.to.(Mirrorer)
if !can {
return "", errors.New("the artifact store here cannot copy an upstream image")
}
return m.MirrorImage(ctx, from, ScratchPrefix+repository)
}
func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
if m, can := s.to.(BaseMirrorer); can {
return m.MirrorBase(ctx, from, repository, former)
}
if m, can := s.to.(Mirrorer); can {
return m.MirrorImage(ctx, from, repository)
}
return "", errors.New("the artifact store here cannot copy a base")
}
// The registry as the scratch namespace.
func (r Registry) store() artifacts.Store {
return artifacts.Store{Address: r.Address, HTTP: r.client()}
}
const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
store := r.store()
for _, b := range builds {
body, err := json.Marshal(b)
if err != nil {
return err
}
if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
}
}
body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
if err != nil {
return err
}
if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
}
return nil
}
// Find reads the record kept under a module's key.
func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
if errors.Is(err, artifacts.ErrNoTag) {
return CheckBuild{}, false, nil
}
if err != nil {
return CheckBuild{}, false, err
}
var b CheckBuild
if err := json.Unmarshal(body, &b); err != nil {
return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
}
if b.Key != key || b.Module != module {
return CheckBuild{}, false, nil
}
return b, true, nil
}
// checks are the check documents kept, by tag; only those of the pull requests named when any are.
func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
store := r.store()
tags, err := store.Tags(ctx, checksRepository)
if err != nil {
return nil, err
}
wanted := map[string]PullRef{}
for _, p := range pulls {
wanted[p.slug()+"."] = p
}
out := map[string]checkIndex{}
for _, tag := range tags {
if len(pulls) > 0 {
match := false
for prefix := range wanted {
match = match || strings.HasPrefix(tag, prefix)
}
if !match {
continue
}
}
body, _, err := store.GetTagged(ctx, checksRepository, tag)
if errors.Is(err, artifacts.ErrNoTag) {
continue
}
if err != nil {
return nil, err
}
var c checkIndex
if err := json.Unmarshal(body, &c); err != nil {
continue // not a document this wrote: left to nobody, said by nothing — it names no build
}
if len(pulls) > 0 {
if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
continue // a slug two pull requests share: only the one named
}
}
out[tag] = c
}
return out, nil
}
// OfPulls are the check builds of those pull requests, newest first.
func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
if len(pulls) == 0 {
return nil, nil
}
checks, err := r.checks(ctx, pulls)
if err != nil {
return nil, err
}
var out []CheckBuild
for _, c := range checks {
out = append(out, c.Builds...)
}
sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
return out, nil
}
// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
repository := module + "/" + made.Name
from := ScratchPrefix + repository
if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
digest := made.Digest
if digest == "" {
_, digest, _ = strings.Cut(made.Reference, "/blobs/")
}
where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
return catalogue.Built{}, err
}
size, err := r.blobSize(ctx, repository, digest)
if err != nil {
return catalogue.Built{}, err
}
if err := r.hold(ctx, repository, digest, int(size)); err != nil {
return catalogue.Built{}, err
}
return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
}
_, digest, pinned := strings.Cut(made.Reference, "@")
if !pinned || !strings.HasPrefix(digest, "sha256:") {
return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
}
reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
if err != nil {
return catalogue.Built{}, err
}
if !strings.HasSuffix(reference, "@"+digest) {
return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
}
return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
}
// blobSize is the size the store says a blob has.
func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
if err != nil {
return 0, err
}
response, err := r.client().Do(request)
if err != nil {
return 0, err
}
response.Body.Close()
if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
}
return response.ContentLength, nil
}
// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
// record under its key when it is still that check's, and the check's document.
func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
checks, err := r.checks(ctx, []PullRef{pull})
if err != nil {
return 0, err
}
return r.letGo(ctx, checks)
}
// Expire lets go of every check older than the bound (§5).
func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
checks, err := r.checks(ctx, nil)
if err != nil {
return 0, err
}
old := map[string]checkIndex{}
for tag, c := range checks {
if now.Sub(c.At) > older {
old[tag] = c
}
}
return r.letGo(ctx, old)
}
// Discard lets go of builds no check document names.
func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
_, err := r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}})
return err
}
// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
// its document, so the next sweep asks again.
func (r Registry) letGo(ctx context.Context, checks map[string]checkIndex) (int, error) {
store := r.store()
gone := 0
var failed []string
for tag, c := range checks {
ok := true
for _, b := range c.Builds {
for _, m := range b.Made {
reference := catalogue.Recorded(m.Reference)
if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
continue // never anything outside the scratch namespace
}
if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
ok = false
failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
}
}
// The record under the key, only while it is still this check's: a later check of the same key
// replaced it, and that one is its own check's to let go of.
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
ok = false
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
}
}
}
if !ok {
continue
}
if tag == "" {
gone++
continue
}
if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
continue
}
gone++
}
if len(failed) > 0 {
sort.Strings(failed)
return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
}
return gone, nil
}
// Reused is what the walk took from a check instead of building (§2, §4).
type Reused struct {
From CheckBuild
// Made are the copies in the module's namespace: what is registered.
Made []catalogue.Built
}
// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
func (r Reused) Words() string {
var digests []string
for _, m := range r.Made {
d := m.Digest
if d == "" {
_, d, _ = strings.Cut(m.Reference, "@")
}
digests = append(digests, m.Name+" "+d)
}
return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
}
internal/builder/reuse_test.go
+541
View File
@@ -0,0 +1,541 @@
package builder
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
type aScratch struct {
builds map[string]CheckBuild
took []string
failTake bool
}
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
for _, b := range builds {
s.builds[b.Module+" "+b.Key] = b
}
return nil
}
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
b, ok := s.builds[module+" "+key]
return b, ok, nil
}
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
var out []CheckBuild
for _, b := range s.builds {
if slices.Contains(pulls, b.Pull) {
out = append(out, b)
}
}
return out, nil
}
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
if s.failTake {
return catalogue.Built{}, errors.New("the store refused the mount")
}
s.took = append(s.took, made.Reference)
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
return made, nil
}
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
const reusedModule = `{"module":"meshboard","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile"},
{"name":"look","kind":"archive","from":"files"}]},
"resources":[
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
t.Helper()
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
return "origin/main\n", nil
}
return r.run(ctx, dir, name, args...)
}
return r, run, t.TempDir()
}
func dockerBuilt(r *recorded) bool {
for _, line := range r.ran {
if strings.HasPrefix(line, "docker build") {
return true
}
}
return false
}
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
t.Helper()
r, run, workspace := aModuleAt(t, tree)
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
Ref: "main"}}}
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
t.Fatalf("the check did not build the module for its merge: %+v", built)
}
if !dockerBuilt(r) {
t.Fatal("the check built no image")
}
for repository := range r.images {
if !strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the check published %s outside the scratch namespace", repository)
}
}
for repository := range r.archives {
if !strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the check published %s outside the scratch namespace", repository)
}
}
return built[0]
}
func withVersion(t *testing.T, v string) {
t.Helper()
was := Version
Version = v
t.Cleanup(func() { Version = was })
}
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
// and registers that digest.
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
}
if got.Reused == nil || got.Why != "" {
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
}
if got.ReuseKey != checked.Key {
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
}
if len(scratch.took) != 2 {
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
}
for _, made := range checked.Made {
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
found := false
for _, b := range got.Built {
found = found || (b.Reference == registered && b.Digest == made.Digest)
}
if !found {
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
}
}
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
}
words := got.Reused.Words()
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
!strings.Contains(words, "sha256:") {
t.Fatalf("the walk says %q", words)
}
}
// A differing key builds, and says which part differs.
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-two")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || got.Reused != nil {
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
}
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
}
// And a builder of another version builds too, saying so.
withVersion(t, "builder-2")
r, run, workspace = aModuleAt(t, "tree-one")
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
}
// No check of the walk's pull requests built it: said.
r, run, workspace = aModuleAt(t, "tree-one")
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
t.Fatalf("the walk says %q", got.Why)
}
}
// A copy that fails is said and falls back to a build; it never fails the walk.
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
aCheckBuilt(t, scratch, "tree-one")
scratch.failTake = true
r, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatalf("a failed copy failed the walk: %v", err)
}
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
t.Fatal("a failed copy did not fall back to a build")
}
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
t.Fatalf("the walk says %q", got.Why)
}
for repository := range r.images {
if strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
}
}
}
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
// off the trunk never takes one.
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-one")
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
Npmrc{}, GitCredential{}, nil); err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || len(scratch.took) > 0 {
t.Fatal("a build not asked to reuse took a check's build")
}
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch}, "https://forge.invalid/meshboard.git",
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
}
}
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
t.Fatal("a check whose repository layer failed builds for its merge")
}
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
t.Fatal("a check that passed with a warning builds nothing for its merge")
}
r, run, workspace := aModuleAt(t, "tree-one")
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
}
}
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
// is not built either.
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
withVersion(t, "builder-1")
r, run, workspace := aModuleAt(t, "tree-one")
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
CheckVerdict{}, GitCredential{}, nil, logging(nil))
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
t.Fatalf("built %+v", built)
}
if len(r.images) > 0 {
t.Fatalf("a module without a key was published: %v", r.images)
}
}
// What the key names: every input a fresh build could differ by.
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
src := newSourceInputs("m")
src.tree = "t1"
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
src.contexts["server"] = "ctx1"
src.toolchains["tools"] = "go sha256:c 1234"
manifest := catalogue.Manifest{Module: "m"}
base := keyOf(src, manifest, "v1")
if base.Key == "" {
t.Fatalf("no key: %s", base.Why)
}
for name, change := range map[string]func(*sourceInputs) string{
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
partBases: func(s *sourceInputs) string {
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
return "v1"
},
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
partBuilder: func(*sourceInputs) string { return "v2" },
} {
s := src.copy()
version := change(s)
if k := keyOf(s, manifest, version); k.Key == base.Key {
t.Errorf("a change of %s left the key the same", name)
}
}
// A base copied to another address is the same base: the key names it by digest.
moved := src.copy()
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
if keyOf(moved, manifest, "v1").Key != base.Key {
t.Error("the same base at another address changed the key")
}
if k := keyOf(src, manifest, ""); k.Key != "" {
t.Error("a builder whose version cannot be said keyed its build")
}
unpinned := src.copy()
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
t.Errorf("a build its source does not pin was keyed: %+v", k)
}
}
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
type aTaggingRegistry struct {
*aRegistryOfRepositories
tags map[string]map[string]string // repository → tag → manifest digest
}
func (m *aTaggingRegistry) serve() http.Handler {
under := m.handler()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
m.mu.Lock()
var tags []string
for tag := range m.tags[repository] {
tags = append(tags, tag)
}
m.mu.Unlock()
if tags == nil {
w.WriteHeader(http.StatusNotFound)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
return
}
repository, kind, rest := splitPath(r.URL.Path)
if kind == "blobs" && r.Method == http.MethodHead {
m.mu.Lock()
body, linked := m.blobs[rest], m.links[repository][rest]
m.mu.Unlock()
if !linked {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
w.WriteHeader(http.StatusOK)
return
}
if kind != "manifests" {
under.ServeHTTP(w, r)
return
}
if !strings.HasPrefix(rest, "sha256:") {
m.mu.Lock()
switch r.Method {
case http.MethodPut:
body, _ := readAll(r)
if m.tags[repository] == nil {
m.tags[repository] = map[string]string{}
}
m.tags[repository][rest] = digestOf(body)
m.manifests[repository+"@"+digestOf(body)] = body
m.mu.Unlock()
w.WriteHeader(http.StatusCreated)
return
default:
digest, ok := m.tags[repository][rest]
body, held := m.manifests[repository+"@"+digest]
m.mu.Unlock()
if !ok || !held {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Docker-Content-Digest", digest)
w.Header().Set("Content-Type", mediaTypeOf(body))
if r.Method == http.MethodGet {
_, _ = w.Write(body)
return
}
w.WriteHeader(http.StatusOK)
return
}
}
if r.Method == http.MethodDelete {
m.mu.Lock()
for _, tags := range m.tags {
for tag, digest := range tags {
if digest == rest {
delete(tags, tag)
}
}
}
m.mu.Unlock()
}
under.ServeHTTP(w, r)
})
}
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
// module's own namespace stays.
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
ctx := t.Context()
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
image := func(repository, body string) string {
digest := digestOf([]byte(body))
m.manifests[repository+"@"+digest] = []byte(body)
return address + "/" + repository + "@" + digest
}
check := func(pull PullRef, id string, at time.Time, key string) string {
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
Reference: reference}}, Pull: pull, Check: id, At: at}
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
t.Fatal(err)
}
return reference
}
held := func(reference string) bool {
_, digest, _ := strings.Cut(reference, "@")
_, ok := m.manifests["scratch/meshboard/server@"+digest]
return ok
}
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
}
n, err := registry.Expire(ctx, CheckBuildsKept, now)
if err != nil || n != 1 {
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
}
if held(old) || !held(fresh) || !held(others) {
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
}
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
t.Fatal("the expired check build's record is still found")
}
n, err = registry.Forget(ctx, aPull)
if err != nil || n != 1 {
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
}
if held(fresh) || !held(others) {
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
}
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
}
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
t.Fatalf("%s still has check builds: %+v", aPull, left)
}
}
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
// in the module's own namespace.
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
layer := []byte("layer")
m.blobs[digestOf(layer)] = layer
m.link("scratch/meshboard/server", digestOf(layer))
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
archive := []byte("archive")
m.blobs[digestOf(archive)] = archive
m.link("scratch/meshboard/look", digestOf(archive))
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
if err != nil {
t.Fatal(err)
}
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
t.Fatalf("copied as %s", got.Reference)
}
if m.uploads != 0 || m.mounts != 1 {
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
}
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
if err != nil {
t.Fatal(err)
}
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
!m.links["meshboard/look"][digestOf(archive)] {
t.Fatalf("the archive was copied as %+v", got)
}
}
internal/builder/source.go
+37
View File
@@ -64,6 +64,43 @@ type sourceInputs struct {
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
// closure is not the one the next merge there meets, and the build says no build source.
contextsAtHead bool
// onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far
// as what its source fingerprint reads, and nothing is built or published.
onlyKey bool
}
// keyOnly is whether these inputs are only being taken for a reuse key.
func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey }
// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own.
func (s *sourceInputs) copy() *sourceInputs {
c := newSourceInputs(s.module)
c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead
c.bases = append([]string{}, s.bases...)
for k, v := range s.contexts {
c.contexts[k] = v
}
for k, v := range s.toolchains {
c.toolchains[k] = v
}
for k, v := range s.own {
c.own[k] = v
}
for k, v := range s.read {
if v == nil {
c.read[k] = nil
continue
}
set := map[string]bool{}
for e := range v {
set[e] = true
}
c.read[k] = set
}
for k, v := range s.readAs {
c.readAs[k] = v
}
return c
}
// contextOffHead says a context was not its trunk's head.
internal/catalogue/verbs.go
+6
View File
@@ -362,6 +362,12 @@ var ControllerVerbs = []Verb{
"register": "\"true\" to register what it builds",
"older": "\"true\", with register: even though a newer build of the module is registered",
}, []string{"id"}, "register", "older")},
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
"when the walk answering its merge ends, and any check build goes after a week.",
Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository",
"number": "the pull request's number"}, []string{"owner", "repo", "number"})},
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
"announces it failed, killed by hand — settled, never handed to another machine.",
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
internal/inventory/plans.go
+3
View File
@@ -286,6 +286,9 @@ type PlanModule struct {
FirstAt *time.Time `json:"first_at,omitempty"`
Commit string `json:"commit,omitempty"`
Why string `json:"why,omitempty"`
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
// request's head, the trunk commit and the digests, or "built: <why>". Said in `plan-moved` and kept here.
Made string `json:"made,omitempty"`
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
// its module's name included. Empty in a plan from before it was kept, which is matched by
internal/link/build.go
+78
View File
@@ -92,6 +92,24 @@ type BuildRequest struct {
// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
// merge-check.sh and answers its verdict. Nothing is built, published or registered.
Check *CheckRequest `json:"check,omitempty"`
// Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key
// of the commit's tree and, where a check build holds the same key, copies it by digest into the module's
// namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a
// differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never
// takes a check's build. A builder from before it ignores it and builds.
Reuse *ReuseAsk `json:"reuse,omitempty"`
}
// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2).
type ReuseAsk struct {
Pulls []PullRef `json:"pulls,omitempty"`
}
// PullRef names a pull request.
type PullRef struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
}
// CheckRequest is what a merge check needs beyond the repository and its head.
@@ -128,6 +146,34 @@ type CheckRequest struct {
// repository's own check is not run for a group: each member's pull request runs its own.
Group string `json:"group,omitempty"`
Members []GroupMember `json:"members,omitempty"`
// Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds
// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
// take. A build seat from before it builds nothing, and the walk builds as before.
Builds []CheckBuild `json:"builds,omitempty"`
}
// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would.
type CheckBuild struct {
Module string `json:"module"`
Repository string `json:"repository,omitempty"`
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
Merged bool `json:"merged,omitempty"`
Tier int `json:"tier"`
// Skip says why the check cannot build it as the walk would — a module that reads the repository checked
// through a context, which a check cannot hand the merge — and nothing after it is built either.
Skip string `json:"skip,omitempty"`
}
// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key
// and the artifacts, in the scratch namespace — or why it was not built.
type CheckBuilt struct {
Module string `json:"module"`
Key string `json:"key,omitempty"`
Made []MadeArtifact `json:"made,omitempty"`
NotBuilt string `json:"not-built,omitempty"`
Took string `json:"took,omitempty"`
}
// GroupMember is one other head of a delivery group's composed check.
@@ -175,6 +221,8 @@ type CheckOutcome struct {
Trunk string `json:"trunk,omitempty"`
Beside map[string]string `json:"beside,omitempty"`
Members []CheckedMember `json:"members,omitempty"`
// Built are the builds it made for its merge (novox/hq ADR 0350 §1).
Built []CheckBuilt `json:"built,omitempty"`
}
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
@@ -264,6 +312,14 @@ type BuildResult struct {
// Failed is why, when it did.
Failed string `json:"failed,omitempty"`
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build
// instead of building, is "reused from <check build>" with its digests, and ReusedFrom which; BuiltWhy, on one
// that built, why it took none (§3, §4). Both empty from a builder that predates it.
ReuseKey string `json:"reuse-key,omitempty"`
Reused string `json:"reused,omitempty"`
ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
BuiltWhy string `json:"built-why,omitempty"`
// Source is the request's, echoed: the seat form of the repository, for whoever registers.
Source *SourceOnSeat `json:"source,omitempty"`
@@ -300,3 +356,25 @@ type MadeArtifact struct {
Kind string `json:"kind"`
Reference string `json:"reference"`
}
// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk
// commit the check merged it into, the check's ask, and its key.
type ReusedFrom struct {
Pull PullRef `json:"pull"`
Head string `json:"head,omitempty"`
Trunk string `json:"trunk,omitempty"`
Check string `json:"check,omitempty"`
Key string `json:"key"`
}
// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from <check build>", or "built:
// <why>". A build seat from before it says neither, and that is said.
func (r BuildResult) MadeWords() string {
switch {
case r.Reused != "":
return r.Reused
case r.BuiltWhy != "":
return "built: " + r.BuiltWhy
}
return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)"
}
internal/link/events.go
+3
View File
@@ -290,6 +290,9 @@ type Checked struct {
Trunk string `json:"trunk,omitempty"`
Beside map[string]string `json:"beside,omitempty"`
Took string `json:"took,omitempty"`
// Built are the builds the check made for its merge, each with its reuse key and digests, or why not
// (novox/hq ADR 0350 §1): what the walk of the merge takes instead of building.
Built []CheckBuilt `json:"built,omitempty"`
}
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
module.json
+1
View File
@@ -76,6 +76,7 @@
"rebuild",
"replay",
"kill",
"forget-checks",
"pause",
"resume",
"hand-act",
testdata/store-tests
Vendored
+1
View File
@@ -59,3 +59,4 @@
./cmd/mesh-controller TestACrossingSendSupersedesASettingsGate
./cmd/mesh-controller TestASettingsWalkFailsOnARefusalThatCannotClear
./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne