jochen
c5663aa18b
An unrecorded index or a copy in progress can name a platform the records do not see, so only a person's collect, after its dry run, takes an index's platforms, and only once every kept index of each repository it touches was read. A copy missing a platform is copied again, and a copy a build holds again is no longer recorded as collected (review of #144).
238 lines
8.0 KiB
Go
238 lines
8.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/artifacts"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// The bases the mesh copied into its artifact store, and the copies no record names (novox/hq ADR 0257).
|
|
//
|
|
// A build that stands on an image published elsewhere copies it in first (ADR 0096, 0097). Since ADR 0257
|
|
// each copy is recorded where it is made, and a copy is kept while a kept build stood on it. Copies made
|
|
// before then by a build that failed — or before the records kept what a build stood on — are in the store
|
|
// with no record naming them, and ADR 0189 §3 keeps the sweep away from anything no record names. This
|
|
// verb is how a person says such a copy is the mesh's: it records it, and the sweep then decides as it
|
|
// does for every other copy. It deletes nothing.
|
|
|
|
// mirrorRepository is a repository only the mirror writes: one image's repository, or a module's own
|
|
// repository of a base from before ADR 0257.
|
|
var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
|
|
|
|
// mirrorReference reads a reference a person gave into its recorded form, refusing anything that is not
|
|
// a manifest in a repository the mirror writes.
|
|
func mirrorReference(given string) (string, error) {
|
|
reference := catalogue.Recorded(strings.TrimSpace(given))
|
|
path, ours := catalogue.InArtifactStore(reference)
|
|
if !ours {
|
|
return "", fmt.Errorf("%q is not a reference into the artifact store (artifact-store://<repository>@sha256:<hex>)", given)
|
|
}
|
|
repository, digest, ok := strings.Cut(path, "@sha256:")
|
|
if !ok || len(digest) != 64 || strings.Trim(digest, "0123456789abcdef") != "" {
|
|
return "", fmt.Errorf("%q names no manifest by digest", given)
|
|
}
|
|
if !mirrorRepository.MatchString(repository) {
|
|
return "", fmt.Errorf("%q is in %s, which is not a repository the mirror writes "+
|
|
"(upstream/<host>/<path>, or <module>/on-<argument>)", given, repository)
|
|
}
|
|
return reference, nil
|
|
}
|
|
|
|
type mirrorEntry struct {
|
|
Reference string `json:"reference"`
|
|
State string `json:"state"`
|
|
Why []string `json:"why,omitempty"`
|
|
}
|
|
|
|
type mirrorsAnswer struct {
|
|
DryRun bool `json:"dry_run,omitempty"`
|
|
// Mirrors is every copy the records hold that is not yet let go of.
|
|
Mirrors []mirrorEntry `json:"mirrors"`
|
|
Counts struct {
|
|
Kept int `json:"kept"`
|
|
Eligible int `json:"eligible"`
|
|
Collected int `json:"collected"`
|
|
} `json:"counts"`
|
|
// Recorded, AlreadyRecorded and Refused answer a record: what was (or, a dry run, would be)
|
|
// recorded, what the records already held, and what was refused, with why.
|
|
Recorded []string `json:"recorded,omitempty"`
|
|
// Then says what recording does: a recorded copy is eligible, and the next sweep lets it go.
|
|
Then string `json:"then,omitempty"`
|
|
AlreadyRecorded []string `json:"already_recorded,omitempty"`
|
|
Refused map[string]string `json:"refused,omitempty"`
|
|
}
|
|
|
|
// mirrorsOf is the copies among the recorded states.
|
|
func mirrorsOf(states []inventory.ArtifactState, mirrored map[string]bool) mirrorsAnswer {
|
|
a := mirrorsAnswer{Mirrors: []mirrorEntry{}}
|
|
for _, s := range states {
|
|
if !mirrored[s.Reference] {
|
|
continue
|
|
}
|
|
switch s.State {
|
|
case inventory.ArtifactKept:
|
|
a.Counts.Kept++
|
|
case inventory.ArtifactEligible:
|
|
a.Counts.Eligible++
|
|
case inventory.ArtifactCollected:
|
|
a.Counts.Collected++
|
|
continue
|
|
}
|
|
a.Mirrors = append(a.Mirrors, mirrorEntry{Reference: s.Reference, State: s.State, Why: s.Why})
|
|
}
|
|
return a
|
|
}
|
|
|
|
// splitReferences reads references separated by spaces or commas.
|
|
func splitReferences(given string) []string {
|
|
return strings.FieldsFunc(given, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
|
}
|
|
|
|
// recordMirrors decides, for each reference given, whether it may be recorded: in a repository the
|
|
// mirror writes, not already recorded, and held by the store. A real run records those.
|
|
func recordMirrors(ctx context.Context, inv *inventory.Inventory, store artifacts.Store, given []string,
|
|
known map[string]bool, why string, real bool) (mirrorsAnswer, error) {
|
|
a := mirrorsAnswer{DryRun: !real, Mirrors: []mirrorEntry{}, Refused: map[string]string{}}
|
|
var record []string
|
|
seen := map[string]bool{}
|
|
for _, g := range given {
|
|
reference, err := mirrorReference(g)
|
|
if err != nil {
|
|
a.Refused[g] = err.Error()
|
|
continue
|
|
}
|
|
if seen[reference] {
|
|
continue
|
|
}
|
|
seen[reference] = true
|
|
if known[reference] {
|
|
a.AlreadyRecorded = append(a.AlreadyRecorded, reference)
|
|
continue
|
|
}
|
|
if store.Address == "" {
|
|
a.Refused[g] = "this mesh has no artifact store on its network to ask whether it holds this"
|
|
continue
|
|
}
|
|
held, err := store.HoldsManifest(ctx, reference)
|
|
switch {
|
|
case err != nil:
|
|
a.Refused[g] = err.Error()
|
|
continue
|
|
case !held:
|
|
a.Refused[g] = "the artifact store does not hold it"
|
|
continue
|
|
}
|
|
record = append(record, reference)
|
|
}
|
|
a.Recorded = record
|
|
if len(record) > 0 {
|
|
verb := "would become"
|
|
if real {
|
|
verb = "became"
|
|
}
|
|
a.Then = fmt.Sprintf("%d cop%s %s eligible: the next sweep (after any build, or collect) lets each go "+
|
|
"unless one of the five kept builds of a module the mesh holds stood on it", len(record),
|
|
map[bool]string{true: "y", false: "ies"}[len(record) == 1], verb)
|
|
}
|
|
if real && len(record) > 0 {
|
|
if err := inv.RecordMirrored(ctx, "", why, record); err != nil {
|
|
return a, fmt.Errorf("recording %d copies: %w", len(record), err)
|
|
}
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
func mirrorsCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("mirrors", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "answer as JSON")
|
|
record := set.String("record", "", "references of copies no record names, separated by spaces or commas, to record as the mesh's")
|
|
confirm := set.Bool("confirm", false, "record them, rather than only say what would be recorded")
|
|
f := addHandActFlags(set)
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 0 {
|
|
return errors.New("mirrors [--json] [--record <references> [--confirm --why <text>]]")
|
|
}
|
|
if *confirm {
|
|
if strings.TrimSpace(*record) == "" {
|
|
return errors.New("mirrors: --confirm records what --record names, and it names nothing. Nothing was done")
|
|
}
|
|
if err := f.require("mirrors"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
states, err := inv.Artifacts(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
known, err := inv.Mirrored(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var answer mirrorsAnswer
|
|
if strings.TrimSpace(*record) == "" {
|
|
answer = mirrorsOf(states, known)
|
|
} else {
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
answer, err = recordMirrors(ctx, inv, artifacts.Store{Address: address}, splitReferences(*record), known,
|
|
strings.TrimSpace(*f.why), *confirm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// The hand act is logged once the records say what it did, never before: an act that
|
|
// failed half-way is not logged as done.
|
|
if *confirm && len(answer.Recorded) > 0 {
|
|
f.record(ctx, "mirrors", append([]string{"--record"}, answer.Recorded...))
|
|
}
|
|
}
|
|
if *asJSON {
|
|
encoder := json.NewEncoder(os.Stdout)
|
|
encoder.SetIndent("", " ")
|
|
return encoder.Encode(answer)
|
|
}
|
|
if answer.DryRun && len(answer.Recorded) > 0 {
|
|
fmt.Println("a dry run: nothing was recorded (--confirm --why <text> to record)")
|
|
}
|
|
if answer.Then != "" {
|
|
fmt.Println(answer.Then)
|
|
}
|
|
for _, r := range answer.Recorded {
|
|
fmt.Printf(" record %s\n", r)
|
|
}
|
|
for _, r := range answer.AlreadyRecorded {
|
|
fmt.Printf(" already %s\n", r)
|
|
}
|
|
for g, why := range answer.Refused {
|
|
fmt.Printf(" refused %s: %s\n", g, why)
|
|
}
|
|
for _, m := range answer.Mirrors {
|
|
fmt.Printf(" %-9s %s %s\n", m.State, m.Reference, strings.Join(m.Why, ", "))
|
|
}
|
|
return nil
|
|
}
|