Record the bases a build copies, keep them by the builds that stood on them, and copy each image once
A copied base was named only in what a build stood on, and nowhere when the build failed, so the store's sweep could never let one go (hq issue 321). One repository per upstream image stops each module asking the public registry for the same image again, and letting an index go now takes its own platform manifests, which otherwise kept every byte. A person can record the copies no record names through the new mirrors verb (hq ADR 0257). The forge test fix is the same commit as on feat/plain-notifications: main fails without it.
This commit is contained in:
jochen
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The bases the mesh copied into its artifact store, and the copies no record names (novox/hq ADR 0257).
|
||||
//
|
||||
// A build that stands on an image published elsewhere copies it in first (ADR 0096, 0097). Since ADR 0257
|
||||
// each copy is recorded where it is made, and a copy is kept while a kept build stood on it. Copies made
|
||||
// before then by a build that failed — or before the records kept what a build stood on — are in the store
|
||||
// with no record naming them, and ADR 0189 §3 keeps the sweep away from anything no record names. This
|
||||
// verb is how a person says such a copy is the mesh's: it records it, and the sweep then decides as it
|
||||
// does for every other copy. It deletes nothing.
|
||||
|
||||
// mirrorRepository is a repository only the mirror writes: one image's repository, or a module's own
|
||||
// repository of a base from before ADR 0257.
|
||||
var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
|
||||
|
||||
// mirrorReference reads a reference a person gave into its recorded form, refusing anything that is not
|
||||
// a manifest in a repository the mirror writes.
|
||||
func mirrorReference(given string) (string, error) {
|
||||
reference := catalogue.Recorded(strings.TrimSpace(given))
|
||||
path, ours := catalogue.InArtifactStore(reference)
|
||||
if !ours {
|
||||
return "", fmt.Errorf("%q is not a reference into the artifact store (artifact-store://<repository>@sha256:<hex>)", given)
|
||||
}
|
||||
repository, digest, ok := strings.Cut(path, "@sha256:")
|
||||
if !ok || len(digest) != 64 || strings.Trim(digest, "0123456789abcdef") != "" {
|
||||
return "", fmt.Errorf("%q names no manifest by digest", given)
|
||||
}
|
||||
if !mirrorRepository.MatchString(repository) {
|
||||
return "", fmt.Errorf("%q is in %s, which is not a repository the mirror writes "+
|
||||
"(upstream/<host>/<path>, or <module>/on-<argument>)", given, repository)
|
||||
}
|
||||
return reference, nil
|
||||
}
|
||||
|
||||
type mirrorEntry struct {
|
||||
Reference string `json:"reference"`
|
||||
State string `json:"state"`
|
||||
Why []string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
type mirrorsAnswer struct {
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
// Mirrors is every copy the records hold that is not yet let go of.
|
||||
Mirrors []mirrorEntry `json:"mirrors"`
|
||||
Counts struct {
|
||||
Kept int `json:"kept"`
|
||||
Eligible int `json:"eligible"`
|
||||
Collected int `json:"collected"`
|
||||
} `json:"counts"`
|
||||
// Recorded, AlreadyRecorded and Refused answer a record: what was (or, a dry run, would be)
|
||||
// recorded, what the records already held, and what was refused, with why.
|
||||
Recorded []string `json:"recorded,omitempty"`
|
||||
AlreadyRecorded []string `json:"already_recorded,omitempty"`
|
||||
Refused map[string]string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// mirrorsOf is the copies among the recorded states.
|
||||
func mirrorsOf(states []inventory.ArtifactState, mirrored map[string]bool) mirrorsAnswer {
|
||||
a := mirrorsAnswer{Mirrors: []mirrorEntry{}}
|
||||
for _, s := range states {
|
||||
if !mirrored[s.Reference] {
|
||||
continue
|
||||
}
|
||||
switch s.State {
|
||||
case inventory.ArtifactKept:
|
||||
a.Counts.Kept++
|
||||
case inventory.ArtifactEligible:
|
||||
a.Counts.Eligible++
|
||||
case inventory.ArtifactCollected:
|
||||
a.Counts.Collected++
|
||||
continue
|
||||
}
|
||||
a.Mirrors = append(a.Mirrors, mirrorEntry{Reference: s.Reference, State: s.State, Why: s.Why})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// splitReferences reads references separated by spaces or commas.
|
||||
func splitReferences(given string) []string {
|
||||
return strings.FieldsFunc(given, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
}
|
||||
|
||||
// recordMirrors decides, for each reference given, whether it may be recorded: in a repository the
|
||||
// mirror writes, not already recorded, and held by the store. A real run records those.
|
||||
func recordMirrors(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, given []string,
|
||||
known map[string]bool, why string, real bool) (mirrorsAnswer, error) {
|
||||
a := mirrorsAnswer{DryRun: !real, Mirrors: []mirrorEntry{}, Refused: map[string]string{}}
|
||||
var record []string
|
||||
seen := map[string]bool{}
|
||||
for _, g := range given {
|
||||
reference, err := mirrorReference(g)
|
||||
if err != nil {
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
}
|
||||
if seen[reference] {
|
||||
continue
|
||||
}
|
||||
seen[reference] = true
|
||||
if known[reference] {
|
||||
a.AlreadyRecorded = append(a.AlreadyRecorded, reference)
|
||||
continue
|
||||
}
|
||||
if store.Address == "" {
|
||||
a.Refused[g] = "this mesh has no artifact store on its network to ask whether it holds this"
|
||||
continue
|
||||
}
|
||||
held, err := store.HoldsManifest(ctx, reference)
|
||||
switch {
|
||||
case err != nil:
|
||||
a.Refused[g] = err.Error()
|
||||
continue
|
||||
case !held:
|
||||
a.Refused[g] = "the artifact store does not hold it"
|
||||
continue
|
||||
}
|
||||
record = append(record, reference)
|
||||
}
|
||||
a.Recorded = record
|
||||
if real && len(record) > 0 {
|
||||
if err := inv.RecordMirrored(ctx, "", why, record); err != nil {
|
||||
return a, fmt.Errorf("recording %d copies: %w", len(record), err)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func mirrorsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("mirrors", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
record := set.String("record", "", "references of copies no record names, separated by spaces or commas, to record as the mesh's")
|
||||
confirm := set.Bool("confirm", false, "record them, rather than only say what would be recorded")
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("mirrors [--json] [--record <references> [--confirm --why <text>]]")
|
||||
}
|
||||
if *confirm {
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
return errors.New("mirrors: --confirm records what --record names, and it names nothing. Nothing was done")
|
||||
}
|
||||
if err := f.require("mirrors"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
states, err := inv.Artifacts(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
known, err := inv.Mirrored(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var answer mirrorsAnswer
|
||||
if strings.TrimSpace(*record) == "" {
|
||||
answer = mirrorsOf(states, known)
|
||||
} else {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *confirm {
|
||||
f.record(ctx, "mirrors", []string{"--record", *record})
|
||||
}
|
||||
answer, err = recordMirrors(ctx, inv, artifacts.Store{Address: address}, splitReferences(*record), known,
|
||||
strings.TrimSpace(*f.why), *confirm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(answer)
|
||||
}
|
||||
if answer.DryRun && len(answer.Recorded) > 0 {
|
||||
fmt.Println("a dry run: nothing was recorded (--confirm --why <text> to record)")
|
||||
}
|
||||
for _, r := range answer.Recorded {
|
||||
fmt.Printf(" record %s\n", r)
|
||||
}
|
||||
for _, r := range answer.AlreadyRecorded {
|
||||
fmt.Printf(" already %s\n", r)
|
||||
}
|
||||
for g, why := range answer.Refused {
|
||||
fmt.Printf(" refused %s: %s\n", g, why)
|
||||
}
|
||||
for _, m := range answer.Mirrors {
|
||||
fmt.Printf(" %-9s %s %s\n", m.State, m.Reference, strings.Join(m.Why, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user