Record the bases a build copies, keep them by the builds that stood on them, and copy each image once
A copied base was named only in what a build stood on, and nowhere when the build failed, so the store's sweep could never let one go (hq issue 321). One repository per upstream image stops each module asking the public registry for the same image again, and letting an index go now takes its own platform manifests, which otherwise kept every byte. A person can record the copies no record names through the new mirrors verb (hq ADR 0257). The forge test fix is the same commit as on feat/plain-notifications: main fails without it.
This commit is contained in:
@@ -77,6 +77,11 @@ type Result struct {
|
||||
// the default, and that branch is its trunk.
|
||||
Branches []string
|
||||
|
||||
// Mirrored is every base this build copied into the artifact store, as pinned (novox/hq ADR 0257):
|
||||
// said whether the build worked or not, because the copy is in the store either way, and the
|
||||
// records are what decide whether it stays.
|
||||
Mirrored []string
|
||||
|
||||
// Source is the build's source fingerprint (source.go): what it was made from — the module's tree,
|
||||
// the contexts' trees, the bases and toolchains by digest — hashed. Empty where the source does not
|
||||
// pin the build. Two builds with one fingerprint are one build, whatever digests they made
|
||||
@@ -106,6 +111,19 @@ type GitCredential struct {
|
||||
func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
|
||||
// **What was copied is said whether the build worked or not** (novox/hq ADR 0257): a base is in
|
||||
// the store from the moment it is copied, and a build that failed after copying it is the
|
||||
// one record that it is there.
|
||||
var mirrored []string
|
||||
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
|
||||
&mirrored, seats...)
|
||||
result.Mirrored = mirrored
|
||||
return result, err
|
||||
}
|
||||
|
||||
func build(ctx context.Context, run Runner, publish Publisher,
|
||||
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
|
||||
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
|
||||
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
|
||||
// that hand none — tests of everything but contexts — read as they did.
|
||||
var seatBases map[string]string
|
||||
@@ -222,10 +240,22 @@ func Build(ctx context.Context, run Runner, publish Publisher,
|
||||
// An image published elsewhere that the build stands on is copied into the mesh's own
|
||||
// registry first, like an upstream artifact (ADR 0096), and the recipe is handed the copy.
|
||||
// Genesis has nowhere to copy to and pulls it into this machine's store instead.
|
||||
mirror := func(ctx context.Context, from, repository string) (string, error) {
|
||||
mirror := func(ctx context.Context, from, repository, former string) (string, error) {
|
||||
if m, can := publish.(BaseMirrorer); can {
|
||||
say("bases", "copying %s into the mesh's registry as %s", from, repository)
|
||||
reference, err := m.MirrorBase(ctx, from, repository, former)
|
||||
if err == nil {
|
||||
*mirrored = append(*mirrored, reference)
|
||||
}
|
||||
return reference, err
|
||||
}
|
||||
if m, can := publish.(Mirrorer); can {
|
||||
say("bases", "copying %s into the mesh's registry", from)
|
||||
return m.MirrorImage(ctx, from, repository)
|
||||
say("bases", "copying %s into the mesh's registry as %s", from, repository)
|
||||
reference, err := m.MirrorImage(ctx, from, repository)
|
||||
if err == nil {
|
||||
*mirrored = append(*mirrored, reference)
|
||||
}
|
||||
return reference, err
|
||||
}
|
||||
if _, err := run(ctx, tree, "docker", "pull", from); err != nil {
|
||||
return "", fmt.Errorf("cannot fetch %s: %w", from, err)
|
||||
@@ -901,7 +931,7 @@ var _ io.Writer = (*stringWriter)(nil)
|
||||
// The order is fixed so two builds of one commit invoke the same command. Returned alongside the
|
||||
// arguments is every reference they resolved to, which is what the build stood on.
|
||||
func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
|
||||
mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
|
||||
mirror func(ctx context.Context, from, repository, former string) (string, error)) ([]string, []string, error) {
|
||||
if manifest.Build == nil || len(manifest.Build.On) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
@@ -924,7 +954,14 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
|
||||
"%s stands on the image %q, which is not pinned by digest. A tag is what "+
|
||||
"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
|
||||
}
|
||||
reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
|
||||
// **One copy per upstream image, whichever modules stand on it** (novox/hq ADR 0257). Its
|
||||
// repository is named for the image, not the module; the module's own repository of
|
||||
// before is offered as a source, so the move to one copy asks upstream for nothing.
|
||||
repository, err := MirrorRepository(base.Image)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
|
||||
}
|
||||
reference, err := mirror(ctx, base.Image, repository, FormerMirrorRepository(manifest.Module, base.Arg))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user