Let platform manifests go only on a confirmed collect, and copy again what a sweep took

An unrecorded index or a copy in progress can name a platform the records do not see, so
only a person's collect, after its dry run, takes an index's platforms, and only once every
kept index of each repository it touches was read. A copy missing a platform is copied
again, and a copy a build holds again is no longer recorded as collected (review of #144).
This commit is contained in:
jochen
2026-10-08 15:47:42 +02:00
parent 9907df6530
commit c5663aa18b
12 changed files with 485 additions and 48 deletions
+42 -2
View File
@@ -232,7 +232,11 @@ func (r Registry) MirrorBase(ctx context.Context, from, repository, former strin
// on the first merge that rebuilt a whole catalogue (2026-09-28), and every module whose base
// lives there failed on a copy it did not need.
if strings.HasPrefix(where.reference, "sha256:") {
held, err := r.has(ctx, "http://"+r.Address+"/v2/"+repository+"/manifests/"+where.reference, manifestAccept)
// **Held whole, not only its index** (novox/hq ADR 0257): every platform manifest the index
// names is asked about too. A sweep that stopped half-way, or that ran while this build was
// copying, may have left an index naming a platform the store no longer holds; such a copy
// is made again, and the copy puts back what is missing.
held, err := r.holdsWhole(ctx, repository, where.reference)
if err != nil {
return "", fmt.Errorf("asking %s whether it holds %s: %w", r.Address, from, err)
}
@@ -242,7 +246,7 @@ func (r Registry) MirrorBase(ctx context.Context, from, repository, former strin
}
src := &source{client: r.client()}
if former != "" && former != repository && strings.HasPrefix(where.reference, "sha256:") {
held, err := r.has(ctx, "http://"+r.Address+"/v2/"+former+"/manifests/"+where.reference, manifestAccept)
held, err := r.holdsWhole(ctx, former, where.reference)
if err != nil {
return "", fmt.Errorf("asking %s whether %s holds %s: %w", r.Address, former, from, err)
}
@@ -259,6 +263,42 @@ func (r Registry) MirrorBase(ctx context.Context, from, repository, former strin
return r.Address + "/" + repository + "@" + digest, nil
}
// holdsWhole is whether this registry holds the manifest under the repository and, for an index, every
// manifest it names.
func (r Registry) holdsWhole(ctx context.Context, repository, digest string) (bool, error) {
url := "http://" + r.Address + "/v2/" + repository + "/manifests/"
held, err := r.has(ctx, url+digest, manifestAccept)
if err != nil || !held {
return false, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url+digest, nil)
if err != nil {
return false, err
}
request.Header.Set("Accept", manifestAccept)
response, err := r.client().Do(request)
if err != nil {
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return false, nil
}
var document struct {
Manifests []descriptor `json:"manifests"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil {
return false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
}
for _, m := range document.Manifests {
held, err := r.has(ctx, url+m.Digest, manifestAccept)
if err != nil || !held {
return false, err
}
}
return true, nil
}
// copyManifest copies one manifest document and everything it names, and returns its digest. An
// index is copied by copying each manifest it names first, so the index never points at something
// the registry does not hold yet.