Merge pull request 'One build per change: the check builds what its merge moves, the walk takes it by key (ADR 0350, issue 649)' (#311) from feat/350-one-build-per-change into main
This commit was merged in pull request #311.
This commit is contained in:
@@ -41,6 +41,9 @@ type holder struct {
|
|||||||
say func(link.HolderState) error
|
say func(link.HolderState) error
|
||||||
// remove runs what a kill needs outside the build: the containers left behind.
|
// remove runs what a kill needs outside the build: the containers left behind.
|
||||||
remove builder.Runner
|
remove builder.Runner
|
||||||
|
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
|
||||||
|
// this machine publishes to no artifact store.
|
||||||
|
scratch builder.Scratch
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
paused bool
|
paused bool
|
||||||
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
|
||||||
|
// after. Every holder sweeps the one store; what one let go of, another finds gone.
|
||||||
|
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
|
||||||
|
sweep := func() {
|
||||||
|
if h.scratch == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
|
||||||
|
case n > 0:
|
||||||
|
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sweep()
|
||||||
|
tick := time.NewTicker(every)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
sweep()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// begin records the build this machine took, with the cancel that ends it.
|
// begin records the build this machine took, with the cancel that ends it.
|
||||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||||
|
|||||||
@@ -97,6 +97,9 @@ func run() error {
|
|||||||
if err := notReady(); err != nil {
|
if err := notReady(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
|
||||||
|
// differently never takes a build another one made.
|
||||||
|
builder.Version = builder.SelfVersion(version)
|
||||||
credential, err := brokerFrom()
|
credential, err := brokerFrom()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -174,6 +177,9 @@ func run() error {
|
|||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||||
|
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
|
||||||
|
h.scratch = publisher
|
||||||
|
go h.expireChecks(ctx, time.Hour, time.Now)
|
||||||
|
|
||||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||||
answer(ctx, publisher, on, workspace, work, h)
|
answer(ctx, publisher, on, workspace, work, h)
|
||||||
@@ -262,6 +268,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
say("packages", saysRegistry(npmrc))
|
say("packages", saysRegistry(npmrc))
|
||||||
}
|
}
|
||||||
|
registryHere, isRegistry := publisher.(builder.Registry)
|
||||||
var built builder.Result
|
var built builder.Result
|
||||||
if request.Check != nil {
|
if request.Check != nil {
|
||||||
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
|
||||||
@@ -271,12 +278,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
if r, ok := publisher.(builder.Registry); ok {
|
if r, ok := publisher.(builder.Registry); ok {
|
||||||
registry = r.Address
|
registry = r.Address
|
||||||
}
|
}
|
||||||
|
spec := checkSpecOf(request)
|
||||||
|
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
|
||||||
|
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
|
||||||
|
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
|
||||||
|
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
|
||||||
|
if isRegistry && len(spec.Builds) > 0 {
|
||||||
|
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
|
||||||
|
Seats: request.Seats}
|
||||||
|
}
|
||||||
var v builder.CheckVerdict
|
var v builder.CheckVerdict
|
||||||
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
|
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
|
||||||
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
|
||||||
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
|
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)}
|
||||||
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
|
||||||
for _, m := range v.Members {
|
for _, m := range v.Members {
|
||||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||||
@@ -289,9 +305,30 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
// after a clone that then fails at npm ci.
|
// after a clone that then fails at npm ci.
|
||||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||||
// docker client (ADR 0219).
|
// docker client (ADR 0219).
|
||||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
if request.Reuse != nil && isRegistry && !request.DryRun {
|
||||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
|
||||||
forgeFrom(), say, request.Seats)
|
reuse := builder.Reuse{Scratch: registryHere}
|
||||||
|
for _, p := range request.Reuse.Pulls {
|
||||||
|
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
|
||||||
|
}
|
||||||
|
// Only what a verdict names is taken (ADR 0350 §2).
|
||||||
|
for _, b := range request.Reuse.Builds {
|
||||||
|
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
|
||||||
|
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
|
||||||
|
Digests: map[string]string{}}
|
||||||
|
for _, m := range b.Built.Made {
|
||||||
|
named.Digests[m.Name] = builder.DigestOf(m.Reference)
|
||||||
|
}
|
||||||
|
reuse.Named = append(reuse.Named, named)
|
||||||
|
}
|
||||||
|
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
|
||||||
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
|
forgeFrom(), say, request.Seats)
|
||||||
|
} else {
|
||||||
|
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||||
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
|
forgeFrom(), say, request.Seats)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
|
||||||
result.Mirrored = built.Mirrored
|
result.Mirrored = built.Mirrored
|
||||||
@@ -327,6 +364,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
}
|
}
|
||||||
result.Against = built.Against
|
result.Against = built.Against
|
||||||
result.SourceFingerprint = built.Source
|
result.SourceFingerprint = built.Source
|
||||||
|
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||||
|
result.ReuseKey = built.ReuseKey
|
||||||
|
if built.Reused != nil {
|
||||||
|
f := built.Reused.From
|
||||||
|
result.Reused = built.Reused.Words()
|
||||||
|
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
|
||||||
|
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
|
||||||
|
} else if request.Reuse != nil {
|
||||||
|
result.BuiltWhy = built.Why
|
||||||
|
if result.BuiltWhy == "" {
|
||||||
|
result.BuiltWhy = "it has no artifact to build"
|
||||||
|
}
|
||||||
|
}
|
||||||
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
|
||||||
for _, r := range built.Read {
|
for _, r := range built.Read {
|
||||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
@@ -377,6 +427,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
for dir, b := range c.Beside {
|
for dir, b := range c.Beside {
|
||||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||||
}
|
}
|
||||||
|
for _, b := range c.Builds {
|
||||||
|
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
|
||||||
|
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
|
||||||
|
}
|
||||||
for _, m := range c.Members {
|
for _, m := range c.Members {
|
||||||
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
|
||||||
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
|
||||||
@@ -384,6 +438,19 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
return spec
|
return spec
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||||
|
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||||
|
var out []link.CheckBuilt
|
||||||
|
for _, b := range built {
|
||||||
|
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()}
|
||||||
|
for _, m := range b.Made {
|
||||||
|
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// layerOf is one layer of a check as the outcome carries it.
|
// layerOf is one layer of a check as the outcome carries it.
|
||||||
func layerOf(l *builder.Layer) *link.CheckLayer {
|
func layerOf(l *builder.Layer) *link.CheckLayer {
|
||||||
if l == nil {
|
if l == nil {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -483,6 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
|||||||
Seats: seatBases(ctx),
|
Seats: seatBases(ctx),
|
||||||
DryRun: dryRun,
|
DryRun: dryRun,
|
||||||
}
|
}
|
||||||
|
request.Reuse = reuseAsked(ctx, dryRun)
|
||||||
fmt.Printf("asked for %s", source)
|
fmt.Printf("asked for %s", source)
|
||||||
if source.Seat != "" {
|
if source.Seat != "" {
|
||||||
fmt.Printf(" (%s)", repository)
|
fmt.Printf(" (%s)", repository)
|
||||||
@@ -669,6 +671,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||||
manifest.Module, short(result.Commit), err)
|
manifest.Module, short(result.Commit), err)
|
||||||
}
|
}
|
||||||
|
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||||
|
// registered only as the walk's copy in the module's own namespace.
|
||||||
|
if scratch := inScratch(result.Made); scratch != "" {
|
||||||
|
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||||
|
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||||
|
}
|
||||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||||
// and the next push would send it.
|
// and the next push would send it.
|
||||||
@@ -714,6 +722,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
return manifest, kept, nil
|
return manifest, kept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||||
|
func inScratch(made []link.MadeArtifact) string {
|
||||||
|
for _, m := range made {
|
||||||
|
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
|
||||||
|
return m.Reference
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
|
||||||
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
|
||||||
"never published (ADR 0238) — merge it, and the merge builds it")
|
"never published (ADR 0238) — merge it, and the merge builds it")
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
|
||||||
|
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
|
||||||
|
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
|
||||||
|
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
|
||||||
|
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
|
||||||
|
// so a word lost here leaves a week's builds, never more.
|
||||||
|
|
||||||
|
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
|
||||||
|
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if address == "" {
|
||||||
|
return nil, errors.New("the artifact store is not on the mesh's network")
|
||||||
|
}
|
||||||
|
return builder.Registry{Address: address}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecks lets go of a pull request's check builds and answers what it did.
|
||||||
|
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||||
|
scratch, err := checkBuildsIn(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
n, err := scratch.Forget(ctx, pull)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
|
||||||
|
// it answers: the walk took what it reuses. Said, never fatal.
|
||||||
|
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
|
||||||
|
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, pull := range reuseAskOf(&p).Pulls {
|
||||||
|
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||||
|
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||||
|
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 3 {
|
||||||
|
return errors.New("forget-checks <owner> <repo> <number>")
|
||||||
|
}
|
||||||
|
number, err := strconv.Atoi(args[2])
|
||||||
|
if err != nil || number <= 0 {
|
||||||
|
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
|
||||||
|
|
||||||
|
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
|
||||||
|
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
|
||||||
|
// that reads the repository through a context not at all. A change that moves no module builds nothing.
|
||||||
|
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
|
||||||
|
entries := []inventory.Entry{
|
||||||
|
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
|
||||||
|
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
|
||||||
|
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
|
||||||
|
}
|
||||||
|
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
|
||||||
|
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
|
||||||
|
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
|
||||||
|
|
||||||
|
got := checkBuildsOf(p, scope, entries, clone)
|
||||||
|
want := []link.CheckBuild{
|
||||||
|
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
|
||||||
|
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
|
||||||
|
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
|
||||||
|
}
|
||||||
|
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
|
||||||
|
t.Fatalf("a change that moves no module asks for builds: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
|
||||||
|
// run never does, and neither does an ask that is not a walk's.
|
||||||
|
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||||
|
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||||
|
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||||
|
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||||
|
asked := reuseAskOf(p)
|
||||||
|
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||||
|
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||||
|
}
|
||||||
|
walk := withReuse(t.Context(), asked)
|
||||||
|
if reuseAsked(walk, false) != asked {
|
||||||
|
t.Fatal("a walk's ask does not carry its leave")
|
||||||
|
}
|
||||||
|
if reuseAsked(walk, true) != nil {
|
||||||
|
t.Fatal("a dry run carries a leave to take a check's build")
|
||||||
|
}
|
||||||
|
if reuseAsked(t.Context(), false) != nil {
|
||||||
|
t.Fatal("an ask that is not a walk's carries a leave")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
|
||||||
|
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
|
||||||
|
var forgot []builder.PullRef
|
||||||
|
was := forgetChecks
|
||||||
|
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
|
||||||
|
forgot = append(forgot, pull)
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { forgetChecks = was })
|
||||||
|
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||||
|
{Repository: "novox/shop", Number: 7}}}}
|
||||||
|
|
||||||
|
p.WalkSaid = inventory.WalkStarted
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
p.WalkSaid = ""
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
if len(forgot) != 0 {
|
||||||
|
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
|
||||||
|
}
|
||||||
|
p.WalkSaid = inventory.WalkEnded
|
||||||
|
forgetChecksOfWalk(t.Context(), nil, p)
|
||||||
|
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
|
||||||
|
t.Fatalf("the walk ended let go of %v", forgot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
|
||||||
|
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
|
||||||
|
t.Fatalf("a module's own build reads as a scratch build: %s", got)
|
||||||
|
}
|
||||||
|
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
|
||||||
|
t.Fatalf("a scratch build was not found: %q", got)
|
||||||
|
}
|
||||||
|
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
|
||||||
|
t.Fatal("a scratch archive was not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
|
||||||
|
// neither, which is said.
|
||||||
|
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||||
|
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
|
||||||
|
t.Fatalf("said %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
|
||||||
|
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
|
||||||
|
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||||
|
// is named in testdata/store-tests.
|
||||||
|
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := g.open.inventory
|
||||||
|
gateEvery = 0
|
||||||
|
|
||||||
|
digest := "sha256:" + strings.Repeat("c", 64)
|
||||||
|
reused := func(id, reference string) link.BuildResult {
|
||||||
|
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
|
||||||
|
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
|
||||||
|
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
|
||||||
|
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
|
||||||
|
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
|
||||||
|
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
|
||||||
|
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
|
||||||
|
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
|
||||||
|
}
|
||||||
|
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "scratch") {
|
||||||
|
t.Fatalf("a build in the scratch namespace was registered: %v", err)
|
||||||
|
}
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Second))
|
||||||
|
result := reused(id, "store.internal:5000/app/server@"+digest)
|
||||||
|
if _, _, err := takeIn(ctx, inv, result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil || current["app"].Commit != "c3" {
|
||||||
|
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(shelf["app"].Resources)
|
||||||
|
if !strings.Contains(string(raw), "app/server@"+digest) {
|
||||||
|
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
|
||||||
|
asked := time.Now().UTC()
|
||||||
|
p := g.plan(t)
|
||||||
|
p.Commit = "c3"
|
||||||
|
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
|
||||||
|
p = g.plan(t)
|
||||||
|
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
|
||||||
|
t.Fatalf("the walk keeps %+v", m)
|
||||||
|
}
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
}
|
||||||
|
p = g.plan(t)
|
||||||
|
gate := p.Modules["app"].Gate
|
||||||
|
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
|
||||||
|
t.Fatalf("the gate judged %+v, not the reused build", gate)
|
||||||
|
}
|
||||||
|
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
|
||||||
|
t.Fatalf("the first machine was sent %v, not the reused build", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A walk's leave names the builds its own pull requests' verdicts made, and none when they cannot be read: the
|
||||||
|
// builder takes nothing a verdict does not name (novox/hq ADR 0350 §2).
|
||||||
|
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||||
|
was := checkedBuildsOf
|
||||||
|
t.Cleanup(func() { checkedBuildsOf = was })
|
||||||
|
var asked []link.PullRef
|
||||||
|
named := []link.CheckedBuild{{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 7}, Check: "build-1",
|
||||||
|
Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}}
|
||||||
|
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
asked = pulls
|
||||||
|
return named, nil
|
||||||
|
}
|
||||||
|
pulls := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}
|
||||||
|
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); !reflect.DeepEqual(got.Builds, named) ||
|
||||||
|
!reflect.DeepEqual(asked, pulls) {
|
||||||
|
t.Fatalf("the leave names %+v, read for %+v", got.Builds, asked)
|
||||||
|
}
|
||||||
|
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
return nil, errors.New("no bus")
|
||||||
|
}
|
||||||
|
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); got.Builds != nil {
|
||||||
|
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -282,10 +282,59 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
Source: sourceOnSeat(source),
|
Source: sourceOnSeat(source),
|
||||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||||
Manifests: scope.Manifests, Judge: scope.Judge},
|
Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
|
||||||
|
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
|
||||||
|
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
|
||||||
|
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
|
||||||
|
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
|
||||||
|
// moves no module builds nothing.
|
||||||
|
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
|
||||||
|
clone func(inventory.Source) (string, error)) []link.CheckBuild {
|
||||||
|
if len(scope.Reach.Plan.Tiers) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
skip := map[string]string{}
|
||||||
|
for _, e := range scope.Reach.Packaging {
|
||||||
|
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
|
||||||
|
"cannot hand the merge"
|
||||||
|
}
|
||||||
|
for _, e := range scope.Reach.Deleted {
|
||||||
|
skip[e.Manifest.Module] = "the merge deletes it"
|
||||||
|
}
|
||||||
|
var out []link.CheckBuild
|
||||||
|
for tier, names := range scope.Reach.Plan.Tiers {
|
||||||
|
for _, name := range names {
|
||||||
|
e, known := byName[name]
|
||||||
|
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
b.Skip = "the catalogue does not hold it"
|
||||||
|
case b.Skip != "":
|
||||||
|
case sourceIs(e.Source, m):
|
||||||
|
b.Merged, b.Path = true, e.Source.Path
|
||||||
|
default:
|
||||||
|
url, err := clone(e.Source)
|
||||||
|
if err != nil {
|
||||||
|
b.Skip = "its repository cannot be cloned: " + err.Error()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
|
||||||
|
}
|
||||||
|
out = append(out, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
|
||||||
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
|
||||||
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
|
||||||
@@ -382,6 +431,7 @@ func checkedOf(result link.BuildResult) link.Checked {
|
|||||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
|
||||||
// repositories read beside it.
|
// repositories read beside it.
|
||||||
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
|
||||||
|
c.Built = result.Check.Built
|
||||||
if c.Gate == nil {
|
if c.Gate == nil {
|
||||||
// A build seat from before the layers: its verdict is the gate's.
|
// A build seat from before the layers: its verdict is the gate's.
|
||||||
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
|
||||||
|
|||||||
@@ -104,6 +104,8 @@ func run() error {
|
|||||||
return replayCommand(ctx, args[1:])
|
return replayCommand(ctx, args[1:])
|
||||||
case "kill":
|
case "kill":
|
||||||
return killCommand(ctx, args[1:])
|
return killCommand(ctx, args[1:])
|
||||||
|
case "forget-checks":
|
||||||
|
return forgetChecksCommand(ctx, args[1:])
|
||||||
case "pause", "resume":
|
case "pause", "resume":
|
||||||
return pauseCommand(ctx, args[0], args[1:])
|
return pauseCommand(ctx, args[0], args[1:])
|
||||||
case "collection":
|
case "collection":
|
||||||
@@ -429,8 +431,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
}
|
}
|
||||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
|
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
|
||||||
|
// before it says nothing, and that is what is kept.
|
||||||
|
made := result.MadeWords()
|
||||||
|
if result.Reused != "" || result.BuiltWhy != "" {
|
||||||
|
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
|
||||||
|
}
|
||||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
|
||||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||||
statusFrom.nudge()
|
statusFrom.nudge()
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -249,7 +249,13 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// (novox/hq to-be 45 §9).
|
// (novox/hq to-be 45 §9).
|
||||||
checkEvents = bus
|
checkEvents = bus
|
||||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
|
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
|
||||||
|
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||||
|
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
|
||||||
|
if p.WalkSaid == inventory.WalkEnded {
|
||||||
|
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := server.Checks(following{open}); err != nil {
|
if err := server.Checks(following{open}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -313,6 +314,86 @@ func init() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
|
||||||
|
type reuseKey struct{}
|
||||||
|
|
||||||
|
// withReuse is ctx carrying a walk's leave to take a check's build.
|
||||||
|
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
|
||||||
|
return context.WithValue(ctx, reuseKey{}, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
|
||||||
|
func reuseIn(ctx context.Context) *link.ReuseAsk {
|
||||||
|
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
|
||||||
|
// takes nothing either (novox/hq ADR 0350 §2, §6).
|
||||||
|
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
|
||||||
|
if dryRun {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return reuseIn(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
|
||||||
|
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
|
||||||
|
r := &link.ReuseAsk{}
|
||||||
|
if p.Delivery == nil {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
seen := map[link.PullRef]bool{}
|
||||||
|
for _, m := range p.Delivery.Merges {
|
||||||
|
pull, ok := pullOf(m)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !seen[pull] {
|
||||||
|
seen[pull] = true
|
||||||
|
r.Pulls = append(r.Pulls, pull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges,
|
||||||
|
// within the week check builds are kept (novox/hq ADR 0350 §2): read from the seat's own outcomes on the bus. A
|
||||||
|
// variable so a test needs no bus.
|
||||||
|
var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||||
|
if len(pulls) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
js, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
|
||||||
|
}
|
||||||
|
|
||||||
|
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
|
||||||
|
// may take (ADR 0350 §2). When they cannot be read the leave names none, and the walk builds, saying why.
|
||||||
|
func reuseWithVerdicts(ctx context.Context, r *link.ReuseAsk) *link.ReuseAsk {
|
||||||
|
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
r.Builds = builds
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
|
||||||
|
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
|
||||||
|
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
|
||||||
|
parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/")
|
||||||
|
if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" {
|
||||||
|
return link.PullRef{}, false
|
||||||
|
}
|
||||||
|
return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true
|
||||||
|
}
|
||||||
|
|
||||||
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
|
||||||
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
|
||||||
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
|
||||||
@@ -340,7 +421,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
||||||
ref = carried
|
ref = carried
|
||||||
}
|
}
|
||||||
id, err := askABuild(ctx, source, e.Source.Path, ref)
|
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||||
|
// requests it answers, whose checks name the part of a differing key.
|
||||||
|
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, reuseAskOf(p))), source, e.Source.Path, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
state.State = "failed"
|
state.State = "failed"
|
||||||
state.Why = err.Error()
|
state.Why = err.Error()
|
||||||
@@ -359,7 +442,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
state.State = "asked"
|
state.State = "asked"
|
||||||
state.AskedAt = &now
|
state.AskedAt = &now
|
||||||
state.Build = id
|
state.Build = id
|
||||||
state.Why, state.Commit, state.BuiltAt = "", "", nil
|
state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
|
||||||
@@ -371,6 +454,12 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
|||||||
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
|
||||||
// build's request time is not known, and such an outcome is taken as before.
|
// build's request time is not known, and such an outcome is taken as before.
|
||||||
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
|
||||||
|
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
|
||||||
|
// ADR 0350 §4) — on the module of the walk it answers.
|
||||||
|
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
|
||||||
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
// than write over what the holder is about to save. Not taken, it is still in the build records,
|
||||||
@@ -436,6 +525,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
|
|||||||
state.State = "built"
|
state.State = "built"
|
||||||
state.BuiltAt = &now
|
state.BuiltAt = &now
|
||||||
state.Commit = commit
|
state.Commit = commit
|
||||||
|
state.Made = made
|
||||||
}
|
}
|
||||||
if err := inv.SavePlan(ctx, p); err != nil {
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
|
||||||
|
|||||||
@@ -452,6 +452,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
case "queue":
|
case "queue":
|
||||||
// As data, the table its output (novox/hq issue 455).
|
// As data, the table its output (novox/hq issue 455).
|
||||||
return []string{"queue", "--json"}, nil
|
return []string{"queue", "--json"}, nil
|
||||||
|
case "forget-checks":
|
||||||
|
if err := need("owner", "repo", "number"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||||
case "cancel", "kill":
|
case "cancel", "kill":
|
||||||
if err := need("id"); err != nil {
|
if err := need("id"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
|
||||||
|
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
|
||||||
|
// or it is seven days old, so the store keeps a week of check builds and no more.
|
||||||
|
|
||||||
|
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
|
||||||
|
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
|
||||||
|
if s.Address == "" {
|
||||||
|
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
|
||||||
|
}
|
||||||
|
var tags []string
|
||||||
|
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
|
||||||
|
for next != "" {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
response, err := s.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||||
|
}
|
||||||
|
switch response.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
case http.StatusNotFound:
|
||||||
|
response.Body.Close()
|
||||||
|
return tags, nil
|
||||||
|
default:
|
||||||
|
response.Body.Close()
|
||||||
|
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
|
||||||
|
}
|
||||||
|
var page struct {
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
|
||||||
|
link := response.Header.Get("Link")
|
||||||
|
response.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
|
||||||
|
}
|
||||||
|
tags = append(tags, page.Tags...)
|
||||||
|
next = nextPage(link, s.Address)
|
||||||
|
}
|
||||||
|
return tags, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
|
||||||
|
func nextPage(link, address string) string {
|
||||||
|
if !strings.Contains(link, `rel="next"`) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
start, end := strings.Index(link, "<"), strings.Index(link, ">")
|
||||||
|
if start < 0 || end <= start+1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
target := link[start+1 : end]
|
||||||
|
if strings.HasPrefix(target, "/") {
|
||||||
|
return "http://" + address + target
|
||||||
|
}
|
||||||
|
return target
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
|
||||||
|
// collection takes its layer. Gone when the tag names nothing.
|
||||||
|
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
|
||||||
|
if s.Address == "" {
|
||||||
|
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
|
||||||
|
}
|
||||||
|
digest, err := s.manifestDigest(ctx, repository, tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if digest == "" {
|
||||||
|
return Gone
|
||||||
|
}
|
||||||
|
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
|
||||||
|
if errors.Is(err, Gone) {
|
||||||
|
return Gone
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
+192
-3
@@ -91,6 +91,16 @@ type Result struct {
|
|||||||
// (novox/hq issue 280).
|
// (novox/hq issue 280).
|
||||||
Source string
|
Source string
|
||||||
|
|
||||||
|
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's
|
||||||
|
// version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build.
|
||||||
|
ReuseKey string
|
||||||
|
ReuseParts map[string]string
|
||||||
|
NoKey string
|
||||||
|
// Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did
|
||||||
|
// not take one (§3). Both empty for a build not asked to reuse.
|
||||||
|
Reused *Reused
|
||||||
|
Why string
|
||||||
|
|
||||||
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
|
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
|
||||||
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
|
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
|
||||||
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
|
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
|
||||||
@@ -136,14 +146,44 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
// one record that it is there.
|
// one record that it is there.
|
||||||
var mirrored []string
|
var mirrored []string
|
||||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||||
&mirrored, seats...)
|
&mirrored, nil, seats...)
|
||||||
|
result.Mirrored = mirrored
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check
|
||||||
|
// builds, and the pull requests the walk answers, whose checks name a differing key.
|
||||||
|
type Reuse struct {
|
||||||
|
Scratch Scratch
|
||||||
|
Pulls []PullRef
|
||||||
|
// Named are the builds the verdicts of the walk's own pull requests name (novox/hq ADR 0350 §2): a scratch build
|
||||||
|
// is taken only when one of them names its key and every digest. The record in the store alone is never enough.
|
||||||
|
Named []NamedBuild
|
||||||
|
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
|
||||||
|
// built at all — nothing could ever take it.
|
||||||
|
keyFirst bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
|
||||||
|
var errNoKey = errors.New("it has no reuse key")
|
||||||
|
|
||||||
|
// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse
|
||||||
|
// key of the commit's tree, and where a check build holds the same key it copies that build into the module's
|
||||||
|
// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3).
|
||||||
|
// Only a commit on its trunk takes a check build: anything else builds, as Build does.
|
||||||
|
func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse,
|
||||||
|
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||||
|
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
|
||||||
|
var mirrored []string
|
||||||
|
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||||
|
&mirrored, &reuse, seats...)
|
||||||
result.Mirrored = mirrored
|
result.Mirrored = mirrored
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func build(ctx context.Context, run Runner, publish Publisher,
|
func build(ctx context.Context, run Runner, publish Publisher,
|
||||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||||
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
|
forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) {
|
||||||
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
|
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
|
||||||
// that hand none — tests of everything but contexts — read as they did.
|
// that hand none — tests of everything but contexts — read as they did.
|
||||||
var seatBases map[string]string
|
var seatBases map[string]string
|
||||||
@@ -284,6 +324,9 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
var built []catalogue.Built
|
var built []catalogue.Built
|
||||||
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
|
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
|
||||||
var stoodOn []string
|
var stoodOn []string
|
||||||
|
// reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350).
|
||||||
|
var reused *Reused
|
||||||
|
builtWhy := ""
|
||||||
if manifest.Build != nil {
|
if manifest.Build != nil {
|
||||||
// What this module said it stands on, answered with what this mesh actually holds. Done
|
// What this module said it stands on, answered with what this mesh actually holds. Done
|
||||||
// before anything is built, so a missing base is refused in front of the person who can
|
// before anything is built, so a missing base is refused in front of the person who can
|
||||||
@@ -327,7 +370,35 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
// Ordered, so two builds of one commit do the same work in the same sequence and their
|
// Ordered, so two builds of one commit do the same work in the same sequence and their
|
||||||
// logs can be compared.
|
// logs can be compared.
|
||||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||||
|
// **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and
|
||||||
|
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
|
||||||
|
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
|
||||||
|
if reuse != nil && reuse.keyFirst {
|
||||||
|
_, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||||
|
held, npmrcPath, npmrc, seatBases, src, say)
|
||||||
|
if err == nil && key.Key == "" {
|
||||||
|
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say("reuse", "not built for the merge: %v", err)
|
||||||
|
return Result{Commit: commit, NoKey: err.Error()}, err
|
||||||
|
}
|
||||||
|
} else if reuse != nil && reuse.Scratch != nil {
|
||||||
|
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
|
||||||
|
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
|
||||||
|
if taken != nil {
|
||||||
|
built = taken.Made
|
||||||
|
src = keyed
|
||||||
|
reused = taken
|
||||||
|
} else {
|
||||||
|
builtWhy = why
|
||||||
|
say("reuse", "built: %s", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, a := range artifacts {
|
for _, a := range artifacts {
|
||||||
|
if reused != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
||||||
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
|
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -363,9 +434,107 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
say("source", "%d path(s) of %s", len(s.Paths), where)
|
say("source", "%d path(s) of %s", len(s.Paths), where)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by.
|
||||||
|
key := keyOf(src, manifest, Version)
|
||||||
return Result{Manifest: resolved, Commit: commit, Built: built,
|
return Result{Manifest: resolved, Commit: commit, Built: built,
|
||||||
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
|
||||||
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil
|
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
|
||||||
|
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
|
||||||
|
// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into
|
||||||
|
// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A
|
||||||
|
// copy that fails is said, and the walk builds (§3): it never fails the walk on its own.
|
||||||
|
func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest,
|
||||||
|
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||||
|
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool,
|
||||||
|
say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) {
|
||||||
|
if !onTrunk {
|
||||||
|
return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build"
|
||||||
|
}
|
||||||
|
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
|
||||||
|
held, npmrcPath, npmrc, seats, src, say)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err.Error()
|
||||||
|
}
|
||||||
|
if key.Key == "" {
|
||||||
|
return nil, nil, "it has no reuse key: " + key.Why
|
||||||
|
}
|
||||||
|
found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, "the scratch namespace could not be read: " + err.Error()
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key)
|
||||||
|
}
|
||||||
|
// **Only a build a verdict names** (ADR 0350 §2): the store takes writes from anything on the network, so a
|
||||||
|
// record is believed only when a check verdict of the walk's own pull requests names its key and its digests.
|
||||||
|
if why := unnamed(reuse.Named, manifest.Module, key.Key, found); why != "" {
|
||||||
|
return nil, nil, why
|
||||||
|
}
|
||||||
|
taken := &Reused{From: found}
|
||||||
|
for _, made := range found.Made {
|
||||||
|
copied, err := reuse.Scratch.Take(ctx, manifest.Module, made)
|
||||||
|
if err != nil {
|
||||||
|
why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err)
|
||||||
|
say("reuse", "%s; built instead", why)
|
||||||
|
return nil, nil, why
|
||||||
|
}
|
||||||
|
taken.Made = append(taken.Made, copied)
|
||||||
|
}
|
||||||
|
// Every artifact the manifest declares, or none: a check build missing one is not this build.
|
||||||
|
if len(taken.Made) != len(artifacts) {
|
||||||
|
return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made),
|
||||||
|
manifest.Module, len(artifacts))
|
||||||
|
}
|
||||||
|
say("reuse", "%s", taken.Words())
|
||||||
|
return taken, keyed, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
|
||||||
|
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
|
||||||
|
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
|
||||||
|
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
|
||||||
|
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs,
|
||||||
|
say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) {
|
||||||
|
keyed := src.copy()
|
||||||
|
keyed.onlyKey = true
|
||||||
|
for _, a := range artifacts {
|
||||||
|
if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held,
|
||||||
|
npmrcPath, npmrc, seats, keyed, say); err != nil {
|
||||||
|
return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keyed.onlyKey = false
|
||||||
|
return keyed, keyOf(keyed, manifest, Version), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the
|
||||||
|
// key differs — naming the part — or the build it made is gone.
|
||||||
|
func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string {
|
||||||
|
if len(reuse.Pulls) == 0 {
|
||||||
|
return "no check built it with this key"
|
||||||
|
}
|
||||||
|
builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls)
|
||||||
|
if err != nil {
|
||||||
|
return "no check build has this key, and which checks built it could not be read: " + err.Error()
|
||||||
|
}
|
||||||
|
for _, b := range builds {
|
||||||
|
if b.Module != module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if b.Key == key.Key {
|
||||||
|
return b.Said() + " built it with this key, and it was removed"
|
||||||
|
}
|
||||||
|
return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ")
|
||||||
|
}
|
||||||
|
var pulls []string
|
||||||
|
for _, p := range reuse.Pulls {
|
||||||
|
pulls = append(pulls, p.String())
|
||||||
|
}
|
||||||
|
return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " +
|
||||||
|
"that did not finish)"
|
||||||
}
|
}
|
||||||
|
|
||||||
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
|
||||||
@@ -754,6 +923,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
|
|
||||||
switch a.Kind {
|
switch a.Kind {
|
||||||
case catalogue.ArtifactUpstream:
|
case catalogue.ArtifactUpstream:
|
||||||
|
// Its key is the image it copies, read from the manifest (keyOf): nothing to take here.
|
||||||
|
if src.keyOnly() {
|
||||||
|
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||||
|
}
|
||||||
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
|
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
|
||||||
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
|
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
|
||||||
//
|
//
|
||||||
@@ -882,6 +1055,10 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
} else {
|
} else {
|
||||||
src.ownWhole()
|
src.ownWhole()
|
||||||
}
|
}
|
||||||
|
// Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built.
|
||||||
|
if src.keyOnly() {
|
||||||
|
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||||
|
}
|
||||||
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
|
||||||
if a.Target != "" {
|
if a.Target != "" {
|
||||||
invocation = append(invocation, "--target", a.Target)
|
invocation = append(invocation, "--target", a.Target)
|
||||||
@@ -949,6 +1126,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
src.notPinned(a.Name + " resolves packages of its own at build time")
|
src.notPinned(a.Name + " resolves packages of its own at build time")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if src.keyOnly() {
|
||||||
|
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||||
|
}
|
||||||
// The module's own packages first, where the compiler and the bundler resolve them from
|
// The module's own packages first, where the compiler and the bundler resolve them from
|
||||||
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
|
||||||
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
|
||||||
@@ -992,6 +1172,9 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
||||||
src.ownWhole()
|
src.ownWhole()
|
||||||
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
|
||||||
|
if src.keyOnly() {
|
||||||
|
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||||
|
}
|
||||||
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
||||||
@@ -1000,6 +1183,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||||
|
|
||||||
case catalogue.ArtifactArchive:
|
case catalogue.ArtifactArchive:
|
||||||
|
if src.keyOnly() {
|
||||||
|
if a.URL == "" {
|
||||||
|
src.ownHas(cleanEntry(a.From) + "/**")
|
||||||
|
}
|
||||||
|
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
|
||||||
|
}
|
||||||
var body []byte
|
var body []byte
|
||||||
var err error
|
var err error
|
||||||
if a.URL != "" {
|
if a.URL != "" {
|
||||||
|
|||||||
@@ -98,6 +98,11 @@ type CheckSpec struct {
|
|||||||
// and composed with it by the gate as one future state. The repository's own check is not run for a
|
// and composed with it by the gate as one future state. The repository's own check is not run for a
|
||||||
// group: each member's pull request runs its own.
|
// group: each member's pull request runs its own.
|
||||||
Group []GroupHead
|
Group []GroupHead
|
||||||
|
// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
|
||||||
|
// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
|
||||||
|
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
||||||
|
Builds []CheckBuildSpec
|
||||||
|
BuildWith *CheckBuilding
|
||||||
}
|
}
|
||||||
|
|
||||||
// GroupHead is one other head of a delivery group's composed check.
|
// GroupHead is one other head of a delivery group's composed check.
|
||||||
@@ -155,6 +160,8 @@ type CheckVerdict struct {
|
|||||||
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
|
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
|
||||||
// commit it was merged into (ADR 0323 §6).
|
// commit it was merged into (ADR 0323 §6).
|
||||||
Members []JudgedHead
|
Members []JudgedHead
|
||||||
|
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
|
||||||
|
Built []CheckBuilt
|
||||||
}
|
}
|
||||||
|
|
||||||
// Layer is one layer of a check, judged.
|
// Layer is one layer of a check, judged.
|
||||||
@@ -572,9 +579,38 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
|
||||||
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
|
||||||
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
|
||||||
|
|
||||||
|
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
|
||||||
|
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
|
||||||
|
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
|
||||||
|
if why := skipsTheBuilds(v); why != "" {
|
||||||
|
say("check", "%s", why)
|
||||||
|
} else {
|
||||||
|
began := time.Now()
|
||||||
|
v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
|
||||||
|
func(step, message string) {
|
||||||
|
if log != nil {
|
||||||
|
log(step, redact.redact(message))
|
||||||
|
}
|
||||||
|
}, say)
|
||||||
|
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
|
||||||
|
time.Since(began).Round(time.Second))
|
||||||
|
}
|
||||||
|
}
|
||||||
return named(v), nil
|
return named(v), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// builtCount is how many of a check's builds for its merge were built.
|
||||||
|
func builtCount(built []CheckBuilt) int {
|
||||||
|
n := 0
|
||||||
|
for _, b := range built {
|
||||||
|
if b.NotBuilt == "" {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
|
||||||
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
|
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
|
||||||
// answers as the context's error.
|
// answers as the context's error.
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
|
||||||
|
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
|
||||||
|
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
|
||||||
|
// it did not fail: a change that fails is not merged, and nothing it built would be taken.
|
||||||
|
|
||||||
|
// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
|
||||||
|
type CheckBuildSpec struct {
|
||||||
|
Module string
|
||||||
|
// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
|
||||||
|
// module is built from the repository checked, at the branch merged into.
|
||||||
|
Repository string
|
||||||
|
Path string
|
||||||
|
Ref string
|
||||||
|
Merged bool
|
||||||
|
Tier int
|
||||||
|
// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
|
||||||
|
Skip string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
|
||||||
|
type CheckBuilding struct {
|
||||||
|
Publish Publisher
|
||||||
|
Scratch Scratch
|
||||||
|
Held map[string]string
|
||||||
|
Npmrc Npmrc
|
||||||
|
Seats map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
|
||||||
|
// why it was not built.
|
||||||
|
type CheckBuilt struct {
|
||||||
|
Module string
|
||||||
|
Key string
|
||||||
|
Made []catalogue.Built
|
||||||
|
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
|
||||||
|
NotBuilt string
|
||||||
|
Took time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
|
||||||
|
func skipsTheBuilds(v CheckVerdict) string {
|
||||||
|
for _, l := range []*Layer{v.Gate, v.Repo} {
|
||||||
|
if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
|
||||||
|
return "the check did not pass, so nothing is built for its merge"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
|
||||||
|
// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
|
||||||
|
// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there.
|
||||||
|
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
|
||||||
|
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
|
||||||
|
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ref := merge
|
||||||
|
if ref == "" {
|
||||||
|
ref = spec.Ref
|
||||||
|
}
|
||||||
|
held := map[string]string{}
|
||||||
|
for k, v := range with.Held {
|
||||||
|
held[k] = v
|
||||||
|
}
|
||||||
|
var out []CheckBuilt
|
||||||
|
var kept []CheckBuild
|
||||||
|
stopAt, stopWhy := -1, ""
|
||||||
|
for _, b := range spec.Builds {
|
||||||
|
began := time.Now()
|
||||||
|
if stopAt >= 0 && b.Tier > stopAt {
|
||||||
|
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if b.Skip != "" {
|
||||||
|
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
|
||||||
|
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
|
||||||
|
if stopAt < 0 || b.Tier < stopAt {
|
||||||
|
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
repository, at := b.Repository, b.Ref
|
||||||
|
if b.Merged {
|
||||||
|
repository, at = tree, ref
|
||||||
|
}
|
||||||
|
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
|
||||||
|
var mirrored []string
|
||||||
|
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
|
||||||
|
filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true},
|
||||||
|
with.Seats)
|
||||||
|
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, errNoKey):
|
||||||
|
built.NotBuilt = err.Error()
|
||||||
|
case err != nil:
|
||||||
|
built.NotBuilt = "its build failed: " + err.Error()
|
||||||
|
case result.ReuseKey == "":
|
||||||
|
built.NotBuilt = "it has no reuse key: " + result.NoKey
|
||||||
|
}
|
||||||
|
if built.NotBuilt != "" {
|
||||||
|
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
|
||||||
|
out = append(out, built)
|
||||||
|
if stopAt < 0 || b.Tier < stopAt {
|
||||||
|
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
built.Made = result.Built
|
||||||
|
for _, m := range result.Built {
|
||||||
|
held[b.Module+"/"+m.Name] = m.Reference
|
||||||
|
}
|
||||||
|
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
|
||||||
|
out = append(out, built)
|
||||||
|
if len(result.Built) > 0 {
|
||||||
|
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
|
||||||
|
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
|
||||||
|
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
|
||||||
|
if len(kept) > 0 {
|
||||||
|
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
|
||||||
|
time.Now().UTC(), kept); err != nil {
|
||||||
|
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
|
||||||
|
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
|
||||||
|
if err := with.Scratch.Discard(ctx, kept); err != nil {
|
||||||
|
say("build", "and what they made could not all be let go of: %v", err)
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
if out[i].NotBuilt == "" {
|
||||||
|
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,608 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One build per change (novox/hq ADR 0350).
|
||||||
|
//
|
||||||
|
// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
|
||||||
|
// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
|
||||||
|
// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
|
||||||
|
// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
|
||||||
|
// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
|
||||||
|
// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
|
||||||
|
// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
|
||||||
|
// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
|
||||||
|
//
|
||||||
|
// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
|
||||||
|
// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
|
||||||
|
// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
|
||||||
|
// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
|
||||||
|
// key, and is always built.
|
||||||
|
|
||||||
|
// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
|
||||||
|
// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
|
||||||
|
// key, every build its own — when it cannot be said.
|
||||||
|
var Version string
|
||||||
|
|
||||||
|
// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
|
||||||
|
// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
|
||||||
|
// builder would read as one).
|
||||||
|
func SelfVersion(stamped string) string {
|
||||||
|
if exe, err := os.Executable(); err == nil {
|
||||||
|
if f, err := os.Open(exe); err == nil {
|
||||||
|
defer f.Close()
|
||||||
|
sum := sha256.New()
|
||||||
|
if _, err := io.Copy(sum, f); err == nil {
|
||||||
|
return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stamped == "" || stamped == "development" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return stamped
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
|
||||||
|
// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
|
||||||
|
const ScratchPrefix = "scratch/"
|
||||||
|
|
||||||
|
// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
|
||||||
|
// What the holder lets go of by (§5).
|
||||||
|
const checksRepository = ScratchPrefix + "checks"
|
||||||
|
|
||||||
|
// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
|
||||||
|
const reuseTagPrefix = "r1-"
|
||||||
|
|
||||||
|
// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
|
||||||
|
const CheckBuildsKept = 7 * 24 * time.Hour
|
||||||
|
|
||||||
|
// The parts of a reuse key, as the walk names the one that differs (§3).
|
||||||
|
const (
|
||||||
|
partModule = "the module"
|
||||||
|
partTree = "the build source"
|
||||||
|
partBases = "a base image"
|
||||||
|
partBuilder = "the builder's version"
|
||||||
|
partContext = "a context"
|
||||||
|
partTool = "a toolchain"
|
||||||
|
partUpstream = "an upstream image"
|
||||||
|
)
|
||||||
|
|
||||||
|
// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
|
||||||
|
type reuseKey struct {
|
||||||
|
Key string
|
||||||
|
Parts map[string]string
|
||||||
|
Why string
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
|
||||||
|
func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
|
||||||
|
if src == nil || src.unpinned != "" || src.tree == "" {
|
||||||
|
why := "its source does not pin the build"
|
||||||
|
if src != nil && src.unpinned != "" {
|
||||||
|
why = src.unpinned
|
||||||
|
} else if src != nil && src.tree == "" {
|
||||||
|
why = "its tree could not be named"
|
||||||
|
}
|
||||||
|
return reuseKey{Why: why}
|
||||||
|
}
|
||||||
|
if version == "" {
|
||||||
|
return reuseKey{Why: "the builder's version cannot be said"}
|
||||||
|
}
|
||||||
|
parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
|
||||||
|
var bases []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, b := range src.bases {
|
||||||
|
if d := referenceDigest(b); !seen[d] {
|
||||||
|
seen[d] = true
|
||||||
|
bases = append(bases, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(bases)
|
||||||
|
parts[partBases] = strings.Join(bases, " ")
|
||||||
|
for a, t := range src.contexts {
|
||||||
|
parts[partContext+" "+a] = t
|
||||||
|
}
|
||||||
|
for a, t := range src.toolchains {
|
||||||
|
parts[partTool+" "+a] = t
|
||||||
|
}
|
||||||
|
// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
|
||||||
|
// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
|
||||||
|
if manifest.Build != nil {
|
||||||
|
for _, a := range manifest.Build.Artifacts {
|
||||||
|
if a.Kind != catalogue.ArtifactUpstream {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.From, "@sha256:") {
|
||||||
|
return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
|
||||||
|
}
|
||||||
|
parts[partUpstream+" "+a.Name] = a.From
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines := make([]string, 0, len(parts))
|
||||||
|
for k, v := range parts {
|
||||||
|
lines = append(lines, k+"="+v)
|
||||||
|
}
|
||||||
|
sort.Strings(lines)
|
||||||
|
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||||
|
return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tagOf is the tag a key's record is kept under.
|
||||||
|
func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
|
||||||
|
|
||||||
|
// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
|
||||||
|
func differing(was, now map[string]string) []string {
|
||||||
|
names := map[string]bool{}
|
||||||
|
for k := range was {
|
||||||
|
names[k] = true
|
||||||
|
}
|
||||||
|
for k := range now {
|
||||||
|
names[k] = true
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for k := range names {
|
||||||
|
if was[k] != now[k] {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullRef names a pull request.
|
||||||
|
type PullRef struct {
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
Repo string `json:"repo"`
|
||||||
|
Number int `json:"number"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
|
||||||
|
|
||||||
|
// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
|
||||||
|
func (p PullRef) slug() string {
|
||||||
|
clean := func(s string) string {
|
||||||
|
return strings.Map(func(r rune) rune {
|
||||||
|
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
return '-'
|
||||||
|
}, s)
|
||||||
|
}
|
||||||
|
return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
|
||||||
|
type CheckBuild struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Parts map[string]string `json:"parts"`
|
||||||
|
// Made are its artifacts, in the scratch namespace.
|
||||||
|
Made []catalogue.Built `json:"made"`
|
||||||
|
// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
|
||||||
|
// check's ask; At when it was built.
|
||||||
|
Pull PullRef `json:"pull"`
|
||||||
|
Head string `json:"head,omitempty"`
|
||||||
|
Trunk string `json:"trunk,omitempty"`
|
||||||
|
Check string `json:"check"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
|
||||||
|
func (c CheckBuild) Said() string {
|
||||||
|
s := "the check of " + c.Pull.String()
|
||||||
|
if c.Head != "" {
|
||||||
|
s += " at head " + twelve(c.Head)
|
||||||
|
}
|
||||||
|
if c.Trunk != "" {
|
||||||
|
s += " (merged into trunk " + twelve(c.Trunk) + ")"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkIndex is one check's document under checksRepository: every build it made.
|
||||||
|
type checkIndex struct {
|
||||||
|
Pull PullRef `json:"pull"`
|
||||||
|
Check string `json:"check"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Builds []CheckBuild `json:"builds"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
|
||||||
|
// Registry is one; a test's is a map.
|
||||||
|
type Scratch interface {
|
||||||
|
// Keep records a check's builds: each under its module and key, and the check's own document.
|
||||||
|
Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
|
||||||
|
// Find is the check build kept under a module's key; false when none is.
|
||||||
|
Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
|
||||||
|
// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
|
||||||
|
// key from.
|
||||||
|
OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
|
||||||
|
// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
|
||||||
|
Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
|
||||||
|
// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
|
||||||
|
// each answers how many checks it let go of.
|
||||||
|
Forget(ctx context.Context, pull PullRef) (int, error)
|
||||||
|
Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
|
||||||
|
// Discard lets go of what builds a check could not keep made: nothing would ever find them.
|
||||||
|
Discard(ctx context.Context, builds []CheckBuild) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
|
||||||
|
// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
|
||||||
|
type scratchPublisher struct{ to Publisher }
|
||||||
|
|
||||||
|
func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
|
||||||
|
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||||
|
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
|
||||||
|
m, can := s.to.(Mirrorer)
|
||||||
|
if !can {
|
||||||
|
return "", errors.New("the artifact store here cannot copy an upstream image")
|
||||||
|
}
|
||||||
|
return m.MirrorImage(ctx, from, ScratchPrefix+repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
|
||||||
|
if m, can := s.to.(BaseMirrorer); can {
|
||||||
|
return m.MirrorBase(ctx, from, repository, former)
|
||||||
|
}
|
||||||
|
if m, can := s.to.(Mirrorer); can {
|
||||||
|
return m.MirrorImage(ctx, from, repository)
|
||||||
|
}
|
||||||
|
return "", errors.New("the artifact store here cannot copy a base")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The registry as the scratch namespace.
|
||||||
|
|
||||||
|
func (r Registry) store() artifacts.Store {
|
||||||
|
return artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||||
|
}
|
||||||
|
|
||||||
|
const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
|
||||||
|
|
||||||
|
// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
|
||||||
|
func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
|
||||||
|
store := r.store()
|
||||||
|
for _, b := range builds {
|
||||||
|
body, err := json.Marshal(b)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
|
||||||
|
return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
|
||||||
|
return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find reads the record kept under a module's key.
|
||||||
|
func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
|
||||||
|
body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
|
||||||
|
if errors.Is(err, artifacts.ErrNoTag) {
|
||||||
|
return CheckBuild{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return CheckBuild{}, false, err
|
||||||
|
}
|
||||||
|
var b CheckBuild
|
||||||
|
if err := json.Unmarshal(body, &b); err != nil {
|
||||||
|
return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
|
||||||
|
}
|
||||||
|
if b.Key != key || b.Module != module {
|
||||||
|
return CheckBuild{}, false, nil
|
||||||
|
}
|
||||||
|
return b, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checks are the check documents kept, by tag; only those of the pull requests named when any are.
|
||||||
|
func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
|
||||||
|
store := r.store()
|
||||||
|
tags, err := store.Tags(ctx, checksRepository)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
wanted := map[string]PullRef{}
|
||||||
|
for _, p := range pulls {
|
||||||
|
wanted[p.slug()+"."] = p
|
||||||
|
}
|
||||||
|
out := map[string]checkIndex{}
|
||||||
|
for _, tag := range tags {
|
||||||
|
if len(pulls) > 0 {
|
||||||
|
match := false
|
||||||
|
for prefix := range wanted {
|
||||||
|
match = match || strings.HasPrefix(tag, prefix)
|
||||||
|
}
|
||||||
|
if !match {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, _, err := store.GetTagged(ctx, checksRepository, tag)
|
||||||
|
if errors.Is(err, artifacts.ErrNoTag) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var c checkIndex
|
||||||
|
if err := json.Unmarshal(body, &c); err != nil {
|
||||||
|
continue // not a document this wrote: left to nobody, said by nothing — it names no build
|
||||||
|
}
|
||||||
|
if len(pulls) > 0 {
|
||||||
|
if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
|
||||||
|
continue // a slug two pull requests share: only the one named
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[tag] = c
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// OfPulls are the check builds of those pull requests, newest first.
|
||||||
|
func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||||
|
if len(pulls) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
checks, err := r.checks(ctx, pulls)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []CheckBuild
|
||||||
|
for _, c := range checks {
|
||||||
|
out = append(out, c.Builds...)
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
|
||||||
|
// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
|
||||||
|
func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||||
|
repository := module + "/" + made.Name
|
||||||
|
from := ScratchPrefix + repository
|
||||||
|
if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
|
||||||
|
digest := made.Digest
|
||||||
|
if digest == "" {
|
||||||
|
_, digest, _ = strings.Cut(made.Reference, "/blobs/")
|
||||||
|
}
|
||||||
|
where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
|
||||||
|
if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
|
||||||
|
return catalogue.Built{}, err
|
||||||
|
}
|
||||||
|
size, err := r.blobSize(ctx, repository, digest)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Built{}, err
|
||||||
|
}
|
||||||
|
if err := r.hold(ctx, repository, digest, int(size)); err != nil {
|
||||||
|
return catalogue.Built{}, err
|
||||||
|
}
|
||||||
|
return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
|
||||||
|
Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
|
||||||
|
}
|
||||||
|
_, digest, pinned := strings.Cut(made.Reference, "@")
|
||||||
|
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||||
|
return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
|
||||||
|
}
|
||||||
|
reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Built{}, err
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(reference, "@"+digest) {
|
||||||
|
return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
|
||||||
|
}
|
||||||
|
return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobSize is the size the store says a blob has.
|
||||||
|
func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
response, err := r.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
response.Body.Close()
|
||||||
|
if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
|
||||||
|
return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
|
||||||
|
}
|
||||||
|
return response.ContentLength, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
|
||||||
|
// record under its key when it is still that check's, and the check's document.
|
||||||
|
func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
|
||||||
|
all, err := r.checks(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
mine := map[string]checkIndex{}
|
||||||
|
for tag, c := range all {
|
||||||
|
if c.Pull == pull {
|
||||||
|
mine[tag] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r.letGo(ctx, mine, all)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expire lets go of every check older than the bound (§5).
|
||||||
|
func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
|
||||||
|
checks, err := r.checks(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
old := map[string]checkIndex{}
|
||||||
|
for tag, c := range checks {
|
||||||
|
if now.Sub(c.At) > older {
|
||||||
|
old[tag] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r.letGo(ctx, old, checks)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard lets go of builds no check document names.
|
||||||
|
func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
|
||||||
|
all, err := r.checks(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}}, all)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
|
||||||
|
// its document, so the next sweep asks again. **A build another check still names stays**: deletion is by digest,
|
||||||
|
// and two checks that built one digest in one repository — a check asked again, a dependent two pull requests move —
|
||||||
|
// share it; it goes with the last check that names it. all are every check kept, the ones going among them.
|
||||||
|
func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex) (int, error) {
|
||||||
|
store := r.store()
|
||||||
|
stays := map[string]bool{}
|
||||||
|
for tag, c := range all {
|
||||||
|
if _, going := checks[tag]; going {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, b := range c.Builds {
|
||||||
|
for _, m := range b.Made {
|
||||||
|
stays[catalogue.Recorded(m.Reference)] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gone := 0
|
||||||
|
var failed []string
|
||||||
|
for tag, c := range checks {
|
||||||
|
ok := true
|
||||||
|
for _, b := range c.Builds {
|
||||||
|
for _, m := range b.Made {
|
||||||
|
reference := catalogue.Recorded(m.Reference)
|
||||||
|
if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
|
||||||
|
continue // never anything outside the scratch namespace
|
||||||
|
}
|
||||||
|
if stays[reference] {
|
||||||
|
continue // another check still names it
|
||||||
|
}
|
||||||
|
if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||||
|
ok = false
|
||||||
|
failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The record under the key, only while it is still this check's: a later check of the same key
|
||||||
|
// replaced it, and that one is its own check's to let go of.
|
||||||
|
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
|
||||||
|
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||||
|
ok = false
|
||||||
|
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if tag == "" {
|
||||||
|
gone++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
|
||||||
|
failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
gone++
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
sort.Strings(failed)
|
||||||
|
return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
|
||||||
|
}
|
||||||
|
return gone, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reused is what the walk took from a check instead of building (§2, §4).
|
||||||
|
type Reused struct {
|
||||||
|
From CheckBuild
|
||||||
|
// Made are the copies in the module's namespace: what is registered.
|
||||||
|
Made []catalogue.Built
|
||||||
|
}
|
||||||
|
|
||||||
|
// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
|
||||||
|
func (r Reused) Words() string {
|
||||||
|
var digests []string
|
||||||
|
for _, m := range r.Made {
|
||||||
|
d := m.Digest
|
||||||
|
if d == "" {
|
||||||
|
_, d, _ = strings.Cut(m.Reference, "@")
|
||||||
|
}
|
||||||
|
digests = append(digests, m.Name+" "+d)
|
||||||
|
}
|
||||||
|
return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// NamedBuild is a build a check's verdict names (ADR 0350 §2): its module, key and each artifact's digest.
|
||||||
|
type NamedBuild struct {
|
||||||
|
Pull PullRef
|
||||||
|
Check string
|
||||||
|
Module string
|
||||||
|
Key string
|
||||||
|
Digests map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DigestOf is the digest a reference names: an image's after `@`, an archive's after `/blobs/`.
|
||||||
|
func DigestOf(reference string) string {
|
||||||
|
if _, d, ok := strings.Cut(reference, "/blobs/"); ok {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
if _, d, ok := strings.Cut(reference, "@"); ok {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// unnamed is why a record found under a key is not taken: no verdict of the walk's pull requests names that module,
|
||||||
|
// key and every one of its digests. Empty when one does.
|
||||||
|
func unnamed(named []NamedBuild, module, key string, found CheckBuild) string {
|
||||||
|
for _, n := range named {
|
||||||
|
if n.Module != module || n.Key != key || n.Check != found.Check || len(n.Digests) != len(found.Made) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
same := true
|
||||||
|
for _, m := range found.Made {
|
||||||
|
d := m.Digest
|
||||||
|
if d == "" {
|
||||||
|
d = DigestOf(m.Reference)
|
||||||
|
}
|
||||||
|
same = same && d != "" && n.Digests[m.Name] == d
|
||||||
|
}
|
||||||
|
if same {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "a build is kept under this key, and no check verdict of the walk's pull requests names it with its " +
|
||||||
|
"digests: not taken"
|
||||||
|
}
|
||||||
@@ -0,0 +1,627 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
|
||||||
|
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
|
||||||
|
|
||||||
|
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
|
||||||
|
type aScratch struct {
|
||||||
|
builds map[string]CheckBuild
|
||||||
|
took []string
|
||||||
|
failTake bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
|
||||||
|
|
||||||
|
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
|
||||||
|
for _, b := range builds {
|
||||||
|
s.builds[b.Module+" "+b.Key] = b
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
|
||||||
|
b, ok := s.builds[module+" "+key]
|
||||||
|
return b, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
||||||
|
var out []CheckBuild
|
||||||
|
for _, b := range s.builds {
|
||||||
|
if slices.Contains(pulls, b.Pull) {
|
||||||
|
out = append(out, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
||||||
|
if s.failTake {
|
||||||
|
return catalogue.Built{}, errors.New("the store refused the mount")
|
||||||
|
}
|
||||||
|
s.took = append(s.took, made.Reference)
|
||||||
|
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||||
|
return made, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
|
||||||
|
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
|
||||||
|
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
|
||||||
|
|
||||||
|
const reusedModule = `{"module":"meshboard","version":"1",
|
||||||
|
"build":{"artifacts":[
|
||||||
|
{"name":"server","kind":"image","from":"Dockerfile"},
|
||||||
|
{"name":"look","kind":"archive","from":"files"}]},
|
||||||
|
"resources":[
|
||||||
|
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
|
||||||
|
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
|
||||||
|
|
||||||
|
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
|
||||||
|
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
|
||||||
|
t.Helper()
|
||||||
|
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
|
||||||
|
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
|
||||||
|
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||||
|
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
|
||||||
|
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
|
||||||
|
return "origin/main\n", nil
|
||||||
|
}
|
||||||
|
return r.run(ctx, dir, name, args...)
|
||||||
|
}
|
||||||
|
return r, run, t.TempDir()
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerBuilt(r *recorded) bool {
|
||||||
|
for _, line := range r.ran {
|
||||||
|
if strings.HasPrefix(line, "docker build") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
|
||||||
|
|
||||||
|
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
|
||||||
|
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
|
||||||
|
t.Helper()
|
||||||
|
r, run, workspace := aModuleAt(t, tree)
|
||||||
|
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
|
||||||
|
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
||||||
|
Ref: "main"}}}
|
||||||
|
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
|
||||||
|
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
|
||||||
|
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
||||||
|
t.Fatalf("the check did not build the module for its merge: %+v", built)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) {
|
||||||
|
t.Fatal("the check built no image")
|
||||||
|
}
|
||||||
|
for repository := range r.images {
|
||||||
|
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||||
|
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for repository := range r.archives {
|
||||||
|
if !strings.HasPrefix(repository, ScratchPrefix) {
|
||||||
|
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return built[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// named is the walk's leave naming what a check's verdict says it built, as the controller reads it.
|
||||||
|
func named(c CheckBuilt) []NamedBuild {
|
||||||
|
n := NamedBuild{Pull: aPull, Check: "build-1", Module: c.Module, Key: c.Key, Digests: map[string]string{}}
|
||||||
|
for _, m := range c.Made {
|
||||||
|
d := m.Digest
|
||||||
|
if d == "" {
|
||||||
|
d = DigestOf(m.Reference)
|
||||||
|
}
|
||||||
|
n.Digests[m.Name] = d
|
||||||
|
}
|
||||||
|
return []NamedBuild{n}
|
||||||
|
}
|
||||||
|
|
||||||
|
func withVersion(t *testing.T, v string) {
|
||||||
|
t.Helper()
|
||||||
|
was := Version
|
||||||
|
Version = v
|
||||||
|
t.Cleanup(func() { Version = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
|
||||||
|
// and registers that digest.
|
||||||
|
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
scratch := newScratch()
|
||||||
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||||
|
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
||||||
|
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
||||||
|
}
|
||||||
|
if got.Reused == nil || got.Why != "" {
|
||||||
|
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
|
||||||
|
}
|
||||||
|
if got.ReuseKey != checked.Key {
|
||||||
|
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
|
||||||
|
}
|
||||||
|
if len(scratch.took) != 2 {
|
||||||
|
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
|
||||||
|
}
|
||||||
|
for _, made := range checked.Made {
|
||||||
|
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
||||||
|
found := false
|
||||||
|
for _, b := range got.Built {
|
||||||
|
found = found || (b.Reference == registered && b.Digest == made.Digest)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
|
||||||
|
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
|
||||||
|
}
|
||||||
|
words := got.Reused.Words()
|
||||||
|
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
|
||||||
|
!strings.Contains(words, "sha256:") {
|
||||||
|
t.Fatalf("the walk says %q", words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything
|
||||||
|
// on its network, so a record planted under another merge's key is never registered.
|
||||||
|
func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
scratch := newScratch()
|
||||||
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||||
|
for name, leave := range map[string][]NamedBuild{
|
||||||
|
"no verdict": nil,
|
||||||
|
"another digest": func() []NamedBuild {
|
||||||
|
n := named(checked)
|
||||||
|
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
||||||
|
return n
|
||||||
|
}(),
|
||||||
|
"another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(),
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
scratch.took = nil
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
|
||||||
|
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests names it") {
|
||||||
|
t.Fatalf("the walk says %q", got.Why)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A differing key builds, and says which part differs.
|
||||||
|
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
scratch := newScratch()
|
||||||
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||||
|
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-two")
|
||||||
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) || got.Reused != nil {
|
||||||
|
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
|
||||||
|
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a builder of another version builds too, saying so.
|
||||||
|
withVersion(t, "builder-2")
|
||||||
|
r, run, workspace = aModuleAt(t, "tree-one")
|
||||||
|
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
|
||||||
|
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No check of the walk's pull requests built it: said.
|
||||||
|
r, run, workspace = aModuleAt(t, "tree-one")
|
||||||
|
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
|
||||||
|
t.Fatalf("the walk says %q", got.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A copy that fails is said and falls back to a build; it never fails the walk.
|
||||||
|
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
scratch := newScratch()
|
||||||
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||||
|
scratch.failTake = true
|
||||||
|
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
||||||
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a failed copy failed the walk: %v", err)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
|
||||||
|
t.Fatal("a failed copy did not fall back to a build")
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
|
||||||
|
t.Fatalf("the walk says %q", got.Why)
|
||||||
|
}
|
||||||
|
for repository := range r.images {
|
||||||
|
if strings.HasPrefix(repository, ScratchPrefix) {
|
||||||
|
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
|
||||||
|
// off the trunk never takes one.
|
||||||
|
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
scratch := newScratch()
|
||||||
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
||||||
|
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
|
||||||
|
Npmrc{}, GitCredential{}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) || len(scratch.took) > 0 {
|
||||||
|
t.Fatal("a build not asked to reuse took a check's build")
|
||||||
|
}
|
||||||
|
|
||||||
|
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
|
||||||
|
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git",
|
||||||
|
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
|
||||||
|
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
|
||||||
|
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
|
||||||
|
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
|
||||||
|
t.Fatal("a check whose repository layer failed builds for its merge")
|
||||||
|
}
|
||||||
|
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
|
||||||
|
t.Fatal("a check that passed with a warning builds nothing for its merge")
|
||||||
|
}
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
|
||||||
|
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
|
||||||
|
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
|
||||||
|
// is not built either.
|
||||||
|
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
|
||||||
|
withVersion(t, "builder-1")
|
||||||
|
r, run, workspace := aModuleAt(t, "tree-one")
|
||||||
|
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
|
||||||
|
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
|
||||||
|
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
|
||||||
|
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
|
||||||
|
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
|
||||||
|
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
|
||||||
|
CheckVerdict{}, GitCredential{}, nil, logging(nil))
|
||||||
|
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
|
||||||
|
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
|
||||||
|
t.Fatalf("built %+v", built)
|
||||||
|
}
|
||||||
|
if len(r.images) > 0 {
|
||||||
|
t.Fatalf("a module without a key was published: %v", r.images)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the key names: every input a fresh build could differ by.
|
||||||
|
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
|
||||||
|
src := newSourceInputs("m")
|
||||||
|
src.tree = "t1"
|
||||||
|
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||||
|
src.contexts["server"] = "ctx1"
|
||||||
|
src.toolchains["tools"] = "go sha256:c 1234"
|
||||||
|
manifest := catalogue.Manifest{Module: "m"}
|
||||||
|
base := keyOf(src, manifest, "v1")
|
||||||
|
if base.Key == "" {
|
||||||
|
t.Fatalf("no key: %s", base.Why)
|
||||||
|
}
|
||||||
|
for name, change := range map[string]func(*sourceInputs) string{
|
||||||
|
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
|
||||||
|
partBases: func(s *sourceInputs) string {
|
||||||
|
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
|
||||||
|
return "v1"
|
||||||
|
},
|
||||||
|
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
|
||||||
|
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
|
||||||
|
partBuilder: func(*sourceInputs) string { return "v2" },
|
||||||
|
} {
|
||||||
|
s := src.copy()
|
||||||
|
version := change(s)
|
||||||
|
if k := keyOf(s, manifest, version); k.Key == base.Key {
|
||||||
|
t.Errorf("a change of %s left the key the same", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A base copied to another address is the same base: the key names it by digest.
|
||||||
|
moved := src.copy()
|
||||||
|
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
|
||||||
|
if keyOf(moved, manifest, "v1").Key != base.Key {
|
||||||
|
t.Error("the same base at another address changed the key")
|
||||||
|
}
|
||||||
|
if k := keyOf(src, manifest, ""); k.Key != "" {
|
||||||
|
t.Error("a builder whose version cannot be said keyed its build")
|
||||||
|
}
|
||||||
|
unpinned := src.copy()
|
||||||
|
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
|
||||||
|
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
|
||||||
|
t.Errorf("a build its source does not pin was keyed: %+v", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
|
||||||
|
type aTaggingRegistry struct {
|
||||||
|
*aRegistryOfRepositories
|
||||||
|
tags map[string]map[string]string // repository → tag → manifest digest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *aTaggingRegistry) serve() http.Handler {
|
||||||
|
under := m.handler()
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
|
||||||
|
m.mu.Lock()
|
||||||
|
var tags []string
|
||||||
|
for tag := range m.tags[repository] {
|
||||||
|
tags = append(tags, tag)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
if tags == nil {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
repository, kind, rest := splitPath(r.URL.Path)
|
||||||
|
if kind == "blobs" && r.Method == http.MethodHead {
|
||||||
|
m.mu.Lock()
|
||||||
|
body, linked := m.blobs[rest], m.links[repository][rest]
|
||||||
|
m.mu.Unlock()
|
||||||
|
if !linked {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if kind != "manifests" {
|
||||||
|
under.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(rest, "sha256:") {
|
||||||
|
m.mu.Lock()
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodPut:
|
||||||
|
body, _ := readAll(r)
|
||||||
|
if m.tags[repository] == nil {
|
||||||
|
m.tags[repository] = map[string]string{}
|
||||||
|
}
|
||||||
|
m.tags[repository][rest] = digestOf(body)
|
||||||
|
m.manifests[repository+"@"+digestOf(body)] = body
|
||||||
|
m.mu.Unlock()
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
digest, ok := m.tags[repository][rest]
|
||||||
|
body, held := m.manifests[repository+"@"+digest]
|
||||||
|
m.mu.Unlock()
|
||||||
|
if !ok || !held {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Docker-Content-Digest", digest)
|
||||||
|
w.Header().Set("Content-Type", mediaTypeOf(body))
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.Method == http.MethodDelete {
|
||||||
|
m.mu.Lock()
|
||||||
|
for _, tags := range m.tags {
|
||||||
|
for tag, digest := range tags {
|
||||||
|
if digest == rest {
|
||||||
|
delete(tags, tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
under.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
|
||||||
|
// module's own namespace stays.
|
||||||
|
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
|
||||||
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||||
|
server := httptest.NewServer(m.serve())
|
||||||
|
defer server.Close()
|
||||||
|
address := strings.TrimPrefix(server.URL, "http://")
|
||||||
|
registry := Registry{Address: address}
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
image := func(repository, body string) string {
|
||||||
|
digest := digestOf([]byte(body))
|
||||||
|
m.manifests[repository+"@"+digest] = []byte(body)
|
||||||
|
return address + "/" + repository + "@" + digest
|
||||||
|
}
|
||||||
|
check := func(pull PullRef, id string, at time.Time, key string) string {
|
||||||
|
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
|
||||||
|
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
|
||||||
|
Reference: reference}}, Pull: pull, Check: id, At: at}
|
||||||
|
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return reference
|
||||||
|
}
|
||||||
|
held := func(reference string) bool {
|
||||||
|
_, digest, _ := strings.Cut(reference, "@")
|
||||||
|
_, ok := m.manifests["scratch/meshboard/server@"+digest]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||||
|
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
|
||||||
|
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
|
||||||
|
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
|
||||||
|
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
|
||||||
|
|
||||||
|
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
|
||||||
|
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
|
||||||
|
}
|
||||||
|
n, err := registry.Expire(ctx, CheckBuildsKept, now)
|
||||||
|
if err != nil || n != 1 {
|
||||||
|
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
|
||||||
|
}
|
||||||
|
if held(old) || !held(fresh) || !held(others) {
|
||||||
|
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
|
||||||
|
}
|
||||||
|
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
|
||||||
|
t.Fatal("the expired check build's record is still found")
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err = registry.Forget(ctx, aPull)
|
||||||
|
if err != nil || n != 1 {
|
||||||
|
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
|
||||||
|
}
|
||||||
|
if held(fresh) || !held(others) {
|
||||||
|
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
|
||||||
|
}
|
||||||
|
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
|
||||||
|
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
|
||||||
|
}
|
||||||
|
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
|
||||||
|
t.Fatalf("%s still has check builds: %+v", aPull, left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
|
||||||
|
// in the module's own namespace.
|
||||||
|
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
|
||||||
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||||
|
server := httptest.NewServer(m.serve())
|
||||||
|
defer server.Close()
|
||||||
|
address := strings.TrimPrefix(server.URL, "http://")
|
||||||
|
registry := Registry{Address: address}
|
||||||
|
|
||||||
|
layer := []byte("layer")
|
||||||
|
m.blobs[digestOf(layer)] = layer
|
||||||
|
m.link("scratch/meshboard/server", digestOf(layer))
|
||||||
|
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
|
||||||
|
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
|
||||||
|
archive := []byte("archive")
|
||||||
|
m.blobs[digestOf(archive)] = archive
|
||||||
|
m.link("scratch/meshboard/look", digestOf(archive))
|
||||||
|
|
||||||
|
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
|
||||||
|
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
|
||||||
|
t.Fatalf("copied as %s", got.Reference)
|
||||||
|
}
|
||||||
|
if m.uploads != 0 || m.mounts != 1 {
|
||||||
|
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
|
||||||
|
}
|
||||||
|
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
|
||||||
|
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
|
||||||
|
!m.links["meshboard/look"][digestOf(archive)] {
|
||||||
|
t.Fatalf("the archive was copied as %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two checks that built one digest in one repository share it: letting go of one check's builds keeps it while the
|
||||||
|
// other still names it, and the last to go takes it.
|
||||||
|
func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
|
||||||
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
||||||
|
server := httptest.NewServer(m.serve())
|
||||||
|
defer server.Close()
|
||||||
|
address := strings.TrimPrefix(server.URL, "http://")
|
||||||
|
registry := Registry{Address: address}
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
||||||
|
digest := digestOf([]byte(body))
|
||||||
|
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
||||||
|
reference := address + "/scratch/meshboard/server@" + digest
|
||||||
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
||||||
|
for i, pull := range []PullRef{aPull, other} {
|
||||||
|
id := "build-" + strconv.Itoa(i+1)
|
||||||
|
b := CheckBuild{Key: "r1:" + strings.Repeat(strconv.Itoa(i+1), 64), Module: "meshboard", Pull: pull, Check: id,
|
||||||
|
At: now, Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: reference}}}
|
||||||
|
if err := registry.Keep(ctx, pull, id, now, []CheckBuild{b}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
||||||
|
|
||||||
|
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
||||||
|
t.Fatalf("forgot %d (%v)", n, err)
|
||||||
|
}
|
||||||
|
if !held() {
|
||||||
|
t.Fatal("letting go of one check's builds took a build another check still names")
|
||||||
|
}
|
||||||
|
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
||||||
|
t.Fatalf("forgot %d (%v)", n, err)
|
||||||
|
}
|
||||||
|
if held() {
|
||||||
|
t.Fatal("the build stayed after the last check naming it went")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -64,6 +64,43 @@ type sourceInputs struct {
|
|||||||
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
|
||||||
// closure is not the one the next merge there meets, and the build says no build source.
|
// closure is not the one the next merge there meets, and the build says no build source.
|
||||||
contextsAtHead bool
|
contextsAtHead bool
|
||||||
|
// onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far
|
||||||
|
// as what its source fingerprint reads, and nothing is built or published.
|
||||||
|
onlyKey bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyOnly is whether these inputs are only being taken for a reuse key.
|
||||||
|
func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey }
|
||||||
|
|
||||||
|
// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own.
|
||||||
|
func (s *sourceInputs) copy() *sourceInputs {
|
||||||
|
c := newSourceInputs(s.module)
|
||||||
|
c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead
|
||||||
|
c.bases = append([]string{}, s.bases...)
|
||||||
|
for k, v := range s.contexts {
|
||||||
|
c.contexts[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range s.toolchains {
|
||||||
|
c.toolchains[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range s.own {
|
||||||
|
c.own[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range s.read {
|
||||||
|
if v == nil {
|
||||||
|
c.read[k] = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
set := map[string]bool{}
|
||||||
|
for e := range v {
|
||||||
|
set[e] = true
|
||||||
|
}
|
||||||
|
c.read[k] = set
|
||||||
|
}
|
||||||
|
for k, v := range s.readAs {
|
||||||
|
c.readAs[k] = v
|
||||||
|
}
|
||||||
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
// contextOffHead says a context was not its trunk's head.
|
// contextOffHead says a context was not its trunk's head.
|
||||||
|
|||||||
@@ -362,6 +362,12 @@ var ControllerVerbs = []Verb{
|
|||||||
"register": "\"true\" to register what it builds",
|
"register": "\"true\" to register what it builds",
|
||||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||||
}, []string{"id"}, "register", "older")},
|
}, []string{"id"}, "register", "older")},
|
||||||
|
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
|
||||||
|
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
|
||||||
|
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
|
||||||
|
"when the walk answering its merge ends, and any check build goes after a week.",
|
||||||
|
Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository",
|
||||||
|
"number": "the pull request's number"}, []string{"owner", "repo", "number"})},
|
||||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||||
|
|||||||
@@ -286,6 +286,9 @@ type PlanModule struct {
|
|||||||
FirstAt *time.Time `json:"first_at,omitempty"`
|
FirstAt *time.Time `json:"first_at,omitempty"`
|
||||||
Commit string `json:"commit,omitempty"`
|
Commit string `json:"commit,omitempty"`
|
||||||
Why string `json:"why,omitempty"`
|
Why string `json:"why,omitempty"`
|
||||||
|
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
|
||||||
|
// request's head, the trunk commit and the digests, or "built: <why>". Said in `plan-moved` and kept here.
|
||||||
|
Made string `json:"made,omitempty"`
|
||||||
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
||||||
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
||||||
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
||||||
|
|||||||
@@ -92,6 +92,38 @@ type BuildRequest struct {
|
|||||||
// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
|
// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
|
||||||
// merge-check.sh and answers its verdict. Nothing is built, published or registered.
|
// merge-check.sh and answers its verdict. Nothing is built, published or registered.
|
||||||
Check *CheckRequest `json:"check,omitempty"`
|
Check *CheckRequest `json:"check,omitempty"`
|
||||||
|
// Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key
|
||||||
|
// of the commit's tree and, where a check build holds the same key, copies it by digest into the module's
|
||||||
|
// namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a
|
||||||
|
// differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never
|
||||||
|
// takes a check's build. A builder from before it ignores it and builds.
|
||||||
|
Reuse *ReuseAsk `json:"reuse,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2).
|
||||||
|
//
|
||||||
|
// **Only a build a verdict names is taken.** Builds are the builds the build seat said, in the outcomes of its checks
|
||||||
|
// of the walk's own pull requests, that it made for their merges: each key with its artifacts' digests. The record
|
||||||
|
// in the scratch namespace is not trusted alone — the artifact store takes writes from anything on the network, a
|
||||||
|
// check runs a pull request's own script on that network, and a record planted under another merge's key would
|
||||||
|
// otherwise be registered. The builder takes a scratch build only when a build here names its key and every digest.
|
||||||
|
type ReuseAsk struct {
|
||||||
|
Pulls []PullRef `json:"pulls,omitempty"`
|
||||||
|
Builds []CheckedBuild `json:"builds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckedBuild is one build a check's verdict names: the pull request it checked, its ask, and the build.
|
||||||
|
type CheckedBuild struct {
|
||||||
|
Pull PullRef `json:"pull"`
|
||||||
|
Check string `json:"check"`
|
||||||
|
Built CheckBuilt `json:"built"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PullRef names a pull request.
|
||||||
|
type PullRef struct {
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
Repo string `json:"repo"`
|
||||||
|
Number int `json:"number"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckRequest is what a merge check needs beyond the repository and its head.
|
// CheckRequest is what a merge check needs beyond the repository and its head.
|
||||||
@@ -128,6 +160,34 @@ type CheckRequest struct {
|
|||||||
// repository's own check is not run for a group: each member's pull request runs its own.
|
// repository's own check is not run for a group: each member's pull request runs its own.
|
||||||
Group string `json:"group,omitempty"`
|
Group string `json:"group,omitempty"`
|
||||||
Members []GroupMember `json:"members,omitempty"`
|
Members []GroupMember `json:"members,omitempty"`
|
||||||
|
// Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds
|
||||||
|
// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
|
||||||
|
// take. A build seat from before it builds nothing, and the walk builds as before.
|
||||||
|
Builds []CheckBuild `json:"builds,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
|
||||||
|
// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would.
|
||||||
|
type CheckBuild struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Repository string `json:"repository,omitempty"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Merged bool `json:"merged,omitempty"`
|
||||||
|
Tier int `json:"tier"`
|
||||||
|
// Skip says why the check cannot build it as the walk would — a module that reads the repository checked
|
||||||
|
// through a context, which a check cannot hand the merge — and nothing after it is built either.
|
||||||
|
Skip string `json:"skip,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key
|
||||||
|
// and the artifacts, in the scratch namespace — or why it was not built.
|
||||||
|
type CheckBuilt struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Key string `json:"key,omitempty"`
|
||||||
|
Made []MadeArtifact `json:"made,omitempty"`
|
||||||
|
NotBuilt string `json:"not-built,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GroupMember is one other head of a delivery group's composed check.
|
// GroupMember is one other head of a delivery group's composed check.
|
||||||
@@ -175,6 +235,8 @@ type CheckOutcome struct {
|
|||||||
Trunk string `json:"trunk,omitempty"`
|
Trunk string `json:"trunk,omitempty"`
|
||||||
Beside map[string]string `json:"beside,omitempty"`
|
Beside map[string]string `json:"beside,omitempty"`
|
||||||
Members []CheckedMember `json:"members,omitempty"`
|
Members []CheckedMember `json:"members,omitempty"`
|
||||||
|
// Built are the builds it made for its merge (novox/hq ADR 0350 §1).
|
||||||
|
Built []CheckBuilt `json:"built,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
|
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
|
||||||
@@ -264,6 +326,14 @@ type BuildResult struct {
|
|||||||
// Failed is why, when it did.
|
// Failed is why, when it did.
|
||||||
Failed string `json:"failed,omitempty"`
|
Failed string `json:"failed,omitempty"`
|
||||||
|
|
||||||
|
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build
|
||||||
|
// instead of building, is "reused from <check build>" with its digests, and ReusedFrom which; BuiltWhy, on one
|
||||||
|
// that built, why it took none (§3, §4). Both empty from a builder that predates it.
|
||||||
|
ReuseKey string `json:"reuse-key,omitempty"`
|
||||||
|
Reused string `json:"reused,omitempty"`
|
||||||
|
ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
|
||||||
|
BuiltWhy string `json:"built-why,omitempty"`
|
||||||
|
|
||||||
// Source is the request's, echoed: the seat form of the repository, for whoever registers.
|
// Source is the request's, echoed: the seat form of the repository, for whoever registers.
|
||||||
Source *SourceOnSeat `json:"source,omitempty"`
|
Source *SourceOnSeat `json:"source,omitempty"`
|
||||||
|
|
||||||
@@ -300,3 +370,25 @@ type MadeArtifact struct {
|
|||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Reference string `json:"reference"`
|
Reference string `json:"reference"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk
|
||||||
|
// commit the check merged it into, the check's ask, and its key.
|
||||||
|
type ReusedFrom struct {
|
||||||
|
Pull PullRef `json:"pull"`
|
||||||
|
Head string `json:"head,omitempty"`
|
||||||
|
Trunk string `json:"trunk,omitempty"`
|
||||||
|
Check string `json:"check,omitempty"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from <check build>", or "built:
|
||||||
|
// <why>". A build seat from before it says neither, and that is said.
|
||||||
|
func (r BuildResult) MadeWords() string {
|
||||||
|
switch {
|
||||||
|
case r.Reused != "":
|
||||||
|
return r.Reused
|
||||||
|
case r.BuiltWhy != "":
|
||||||
|
return "built: " + r.BuiltWhy
|
||||||
|
}
|
||||||
|
return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)"
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only
|
||||||
|
// checks of the walk's pull requests, only builds made, newest first.
|
||||||
|
func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) {
|
||||||
|
js := aBus(t)
|
||||||
|
since := time.Now().Add(-time.Minute)
|
||||||
|
say := func(r BuildResult) {
|
||||||
|
body, _ := json.Marshal(r)
|
||||||
|
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
|
||||||
|
built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
|
||||||
|
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built,
|
||||||
|
{Module: "web", NotBuilt: "it has no reuse key"}}}})
|
||||||
|
say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8},
|
||||||
|
Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}})
|
||||||
|
say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check
|
||||||
|
newer := built
|
||||||
|
newer.Key = "r1:2"
|
||||||
|
say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}})
|
||||||
|
|
||||||
|
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" ||
|
||||||
|
got[1].Pull.Number != 7 {
|
||||||
|
t.Fatalf("read %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -290,6 +290,9 @@ type Checked struct {
|
|||||||
Trunk string `json:"trunk,omitempty"`
|
Trunk string `json:"trunk,omitempty"`
|
||||||
Beside map[string]string `json:"beside,omitempty"`
|
Beside map[string]string `json:"beside,omitempty"`
|
||||||
Took string `json:"took,omitempty"`
|
Took string `json:"took,omitempty"`
|
||||||
|
// Built are the builds the check made for its merge, each with its reuse key and digests, or why not
|
||||||
|
// (novox/hq ADR 0350 §1): what the walk of the merge takes instead of building.
|
||||||
|
Built []CheckBuilt `json:"built,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
|
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
|
||||||
|
|||||||
@@ -364,6 +364,64 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
|
|||||||
return heard, nil
|
return heard, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadCheckBuilds is every build the seat's checks of those pull requests said they made for their merges since a
|
||||||
|
// moment (novox/hq ADR 0350 §2): read from the seat's own outcomes on the events stream, which only its holders may
|
||||||
|
// publish, never from the artifact store. Newest first.
|
||||||
|
func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pulls []PullRef, since time.Time) ([]CheckedBuild, error) {
|
||||||
|
if len(pulls) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
wanted := map[PullRef]bool{}
|
||||||
|
for _, p := range pulls {
|
||||||
|
wanted[p] = true
|
||||||
|
}
|
||||||
|
subject := BuildOutcomeOf(seat)
|
||||||
|
sub, err := js.Context().PullSubscribe(subject, "",
|
||||||
|
nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
info, err := sub.ConsumerInfo()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||||
|
}
|
||||||
|
var out []CheckedBuild
|
||||||
|
for remaining := info.NumPending; remaining > 0; {
|
||||||
|
batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
|
||||||
|
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return nil, fmt.Errorf("reading %s: %w", subject, err)
|
||||||
|
}
|
||||||
|
if len(batch) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
remaining -= uint64(len(batch))
|
||||||
|
for _, msg := range batch {
|
||||||
|
var r BuildResult
|
||||||
|
if json.Unmarshal(msg.Data, &r) != nil || r.Checked == nil || r.Check == nil || r.Failed != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pull := PullRef{Owner: r.Checked.Owner, Repo: r.Checked.Repo, Number: r.Checked.Number}
|
||||||
|
if !wanted[pull] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, b := range r.Check.Built {
|
||||||
|
if b.NotBuilt == "" && b.Key != "" {
|
||||||
|
out = append(out, CheckedBuild{Pull: pull, Check: r.ID, Built: b})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Newest first: the stream's order is the order they were said.
|
||||||
|
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
out[i], out[j] = out[j], out[i]
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// --- the cancelled set ----------------------------------------------------------------------
|
// --- the cancelled set ----------------------------------------------------------------------
|
||||||
|
|
||||||
// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
|
// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
|
||||||
|
|||||||
@@ -76,6 +76,7 @@
|
|||||||
"rebuild",
|
"rebuild",
|
||||||
"replay",
|
"replay",
|
||||||
"kill",
|
"kill",
|
||||||
|
"forget-checks",
|
||||||
"pause",
|
"pause",
|
||||||
"resume",
|
"resume",
|
||||||
"hand-act",
|
"hand-act",
|
||||||
|
|||||||
Vendored
+1
@@ -59,3 +59,4 @@
|
|||||||
./cmd/mesh-controller TestACrossingSendSupersedesASettingsGate
|
./cmd/mesh-controller TestACrossingSendSupersedesASettingsGate
|
||||||
./cmd/mesh-controller TestASettingsWalkFailsOnARefusalThatCannotClear
|
./cmd/mesh-controller TestASettingsWalkFailsOnARefusalThatCannotClear
|
||||||
./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
|
./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
|
||||||
|
./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne
|
||||||
|
|||||||
Reference in New Issue
Block a user