package main import ( "context" "errors" "fmt" "path" "sort" "strings" "sync" "time" ) // Retention for the package registry (novox/hq ADR 0251 §4, to-be 51): a version is kept when a // lockfile on a repository's default branch names it, when it is the highest version satisfying a // range a package manifest there names, when a dist-tag names it, or when it is among the newest // `keep` of its package. Everything else is what retention would delete. // DefaultKeep is how many of each package's newest versions are kept whatever names them: the same // five builds the artifact store keeps (ADR 0189 §3), so "how far back" has one answer. const DefaultKeep = 5 // workers bounds how many calls go to the forge at once; budget how long one tool call may read. const ( workers = 8 budget = 50 * time.Second ) // Item is one version, with what keeps it. type Item struct { Version string `json:"version"` Published time.Time `json:"published"` Bytes int64 `json:"bytes"` SizeError string `json:"size_error,omitempty"` Kept bool `json:"kept"` Why []string `json:"why,omitempty"` } // Package is one package's versions, newest first. type Package struct { Name string `json:"name"` DistTags map[string]string `json:"dist_tags,omitempty"` // KeepsAll says why every version is kept, when something about the package could not be read. KeepsAll string `json:"keeps_all,omitempty"` Versions []Item `json:"versions"` } // Survey is the registry and what the forge's repositories say of it. type Survey struct { Owner string `json:"owner"` Keep int `json:"keep"` Repositories int `json:"repositories_read"` Unread []string `json:"repositories_unread,omitempty"` NotRead []string `json:"files_not_read,omitempty"` Packages []Package `json:"packages"` Said []string `json:"said"` } // survey reads the registry and every repository, and decides what is kept. sizes says whether to // ask the forge each version's size (for these versions only, when not nil). func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Item) bool) (*Survey, error) { ctx, cancel := context.WithTimeout(ctx, budget) defer cancel() listed, err := f.Packages(ctx) if err != nil { return nil, err } byName := map[string][]PackageVersion{} for _, v := range listed { byName[v.Name] = append(byName[v.Name], v) } if only != "" { if _, has := byName[only]; !has { return nil, fmt.Errorf("the registry of %s holds no npm package %q", f.Owner, only) } } known := map[string]bool{} for name := range byName { known[name] = true } s := &Survey{Owner: f.Owner, Keep: keep, Said: []string{}} wanted, pinned, err := readRepositories(ctx, f, known, s) if err != nil { return nil, err } names := make([]string, 0, len(byName)) for name := range byName { if only == "" || name == only { names = append(names, name) } } sort.Strings(names) tags := distTags(ctx, f, names) for _, name := range names { s.Packages = append(s.Packages, decide(name, byName[name], tags[name], wanted[name], pinned[name], keep)) } if sizes != nil { measure(ctx, f, s, sizes) } return s, nil } type tagAnswer struct { tags map[string]string err error } func distTags(ctx context.Context, f *Forge, names []string) map[string]tagAnswer { out := map[string]tagAnswer{} var mu sync.Mutex each(names, func(name string) { t, err := f.DistTags(ctx, name) mu.Lock() out[name] = tagAnswer{t, err} mu.Unlock() }) return out } // each runs fn over items, workers at a time. func each[T any](items []T, fn func(T)) { var wg sync.WaitGroup gate := make(chan struct{}, workers) for _, item := range items { wg.Add(1) gate <- struct{}{} go func(item T) { defer wg.Done() defer func() { <-gate }() fn(item) }(item) } wg.Wait() } // readRepositories reads every repository's manifests and lockfiles. A repository that could not be // read is named in Unread: what it depends on is not known, so nothing may be deleted. func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *Survey) (map[string][]Wanted, map[string][]Pinned, error) { repos, err := f.Repositories(ctx) if err != nil { return nil, nil, fmt.Errorf("the forge's repositories cannot be listed, so nothing is known to depend on anything: %w", err) } wanted := map[string][]Wanted{} pinned := map[string][]Pinned{} var mu sync.Mutex each(repos, func(r Repository) { if r.Empty || r.DefaultBranch == "" { mu.Lock() s.Repositories++ mu.Unlock() return } files, err := f.Files(ctx, r) if errors.Is(err, errNotFound) { // A repository without its default branch holds nothing to read. files, err = nil, nil } if err != nil { mu.Lock() s.Unread = append(s.Unread, fmt.Sprintf("%s: %v", r.FullName, err)) mu.Unlock() return } var ws []Wanted var ps []Pinned var notRead, unread []string for _, p := range files { if !interesting(p) { continue } where := r.FullName + ":" + p content, err := f.Raw(ctx, r, p) if err != nil { unread = append(unread, fmt.Sprintf("%s: %v", where, err)) continue } base := path.Base(p) if manifestNames[base] { got, err := ReadManifest(content, where, known) if err != nil { notRead = append(notRead, fmt.Sprintf("%s: not JSON (%v)", where, err)) continue } ws = append(ws, got...) continue } got, err := ReadLock(base, content, where, known) ps = append(ps, got...) if err != nil { notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err)) } } mu.Lock() defer mu.Unlock() s.Repositories++ s.Unread = append(s.Unread, unread...) s.NotRead = append(s.NotRead, notRead...) for _, w := range ws { wanted[w.Package] = append(wanted[w.Package], w) } for _, p := range ps { pinned[p.Package] = append(pinned[p.Package], p) } }) sort.Strings(s.Unread) sort.Strings(s.NotRead) return wanted, pinned, nil } // decide is one package's versions, newest first, each with what keeps it. func decide(name string, listed []PackageVersion, tags tagAnswer, wanted []Wanted, pinned []Pinned, keep int) Package { p := Package{Name: name, DistTags: tags.tags} type entry struct { item Item v Version ok bool } var entries []*entry byVersion := map[string]*entry{} var parsed []Version for _, l := range listed { e := &entry{item: Item{Version: l.Version, Published: l.CreatedAt}} e.v, e.ok = mustVersion(l.Version) entries = append(entries, e) byVersion[l.Version] = e if e.ok { parsed = append(parsed, e.v) } } why := func(version, reason string) { if e := byVersion[version]; e != nil { e.item.Why = append(e.item.Why, reason) } } var keepsAll []string if tags.err != nil { keepsAll = append(keepsAll, fmt.Sprintf("its dist-tags could not be read (%v)", tags.err)) } for tag, version := range tags.tags { why(version, "dist-tag "+tag) } for _, pin := range pinned { why(pin.Version, "a lockfile names it: "+pin.Where) } for _, w := range wanted { if version, isTag := tags.tags[w.Range]; isTag { why(version, fmt.Sprintf("dist-tag %q is the range %s names", w.Range, w.Where)) continue } r, err := ParseRange(w.Range) if err != nil { keepsAll = append(keepsAll, fmt.Sprintf("the range %q in %s cannot be read (%v)", w.Range, w.Where, err)) continue } if best, found := MaxSatisfying(r, parsed); found { why(best.String(), fmt.Sprintf("the highest version satisfying %s in %s", w.Range, w.Where)) } } sort.SliceStable(entries, func(i, j int) bool { a, b := entries[i], entries[j] switch { case a.ok && b.ok: return Compare(a.v, b.v) > 0 case a.ok != b.ok: return a.ok // a version that is not semver sorts last } return a.item.Published.After(b.item.Published) }) newest := 0 for _, e := range entries { if !e.ok { e.item.Why = append(e.item.Why, "not a semantic version, so not ordered: kept") continue } if newest < keep { newest++ e.item.Why = append(e.item.Why, fmt.Sprintf("among the newest %d (pre-releases count)", keep)) } } if len(keepsAll) > 0 { p.KeepsAll = strings.Join(keepsAll, "; ") } for _, e := range entries { e.item.Why = dedupe(e.item.Why) e.item.Kept = len(e.item.Why) > 0 || p.KeepsAll != "" p.Versions = append(p.Versions, e.item) } return p } func mustVersion(s string) (Version, bool) { v, err := ParseVersion(s) return v, err == nil } // dedupe keeps each reason once and at most a few of each kind, so a version a hundred lockfiles name // is not a hundred lines. func dedupe(why []string) []string { seen := map[string]bool{} kinds := map[string]int{} var out []string more := map[string]int{} for _, w := range why { if seen[w] { continue } seen[w] = true kind, _, _ := strings.Cut(w, ":") if kinds[kind] >= 3 { more[kind]++ continue } kinds[kind]++ out = append(out, w) } for kind, n := range more { out = append(out, fmt.Sprintf("%s: and %d more", kind, n)) } sort.Strings(out) return out } // measure asks the forge the size of each version sizes picks. func measure(ctx context.Context, f *Forge, s *Survey, sizes func(Item) bool) { type at struct{ p, v int } var todo []at for i := range s.Packages { for j := range s.Packages[i].Versions { if sizes(s.Packages[i].Versions[j]) { todo = append(todo, at{i, j}) } } } var mu sync.Mutex each(todo, func(a at) { p := &s.Packages[a.p] n, err := f.Size(ctx, p.Name, p.Versions[a.v].Version) mu.Lock() defer mu.Unlock() if err != nil { p.Versions[a.v].SizeError = err.Error() return } p.Versions[a.v].Bytes = n }) } // Candidate is one version retention would delete. type Candidate struct { Package string `json:"package"` Version string `json:"version"` Published time.Time `json:"published"` Bytes int64 `json:"bytes"` SizeError string `json:"size_error,omitempty"` } // Plan is what retention would delete, and — for a real run — what it did. type Plan struct { *Survey DryRun bool `json:"dry_run"` Why string `json:"why,omitempty"` Candidates []Candidate `json:"would_delete"` Bytes int64 `json:"bytes"` // RegistryBytes is every version's size together, deleted or kept: what retention is measured against. RegistryBytes int64 `json:"registry_bytes"` Deleted []string `json:"deleted,omitempty"` Refused []string `json:"refused,omitempty"` } // retention works out the plan, and carries it out when dryRun is false. func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool, why string) (*Plan, error) { if !dryRun && strings.TrimSpace(why) == "" { return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done") } // Every version is measured, kept or not, so no size in the answer is a 0 nobody asked for. s, err := survey(ctx, f, only, keep, func(Item) bool { return true }) if err != nil { return nil, err } p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}} for _, pkg := range s.Packages { for _, v := range pkg.Versions { p.RegistryBytes += v.Bytes if !v.Kept { p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes, v.SizeError}) p.Bytes += v.Bytes } } } s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages, %s; %d would be deleted, %s", s.Repositories, countVersions(s), len(s.Packages), mib(p.RegistryBytes), len(p.Candidates), mib(p.Bytes))) s.Said = append(s.Said, sizesSaid(s)...) if len(s.NotRead) > 0 { s.Said = append(s.Said, fmt.Sprintf("%d files were not read whole (files_not_read): what they name is not "+ "known, so a real run deletes nothing", len(s.NotRead))) } if len(s.Unread) > 0 { s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+ "known, so a real run deletes nothing", len(s.Unread))) } if dryRun { s.Said = append(s.Said, "a dry run: nothing was deleted") return p, nil } if len(s.Unread) > 0 { return p, fmt.Errorf("refused: %d repositories or files could not be read, so what depends on what is not "+ "known; nothing was deleted (first: %s)", len(s.Unread), s.Unread[0]) } // A file that was fetched but not read whole is a file of its repository that could not be read // (novox/hq ADR 0251 §4): a version only it names would be deleted while in use. if len(s.NotRead) > 0 { return p, fmt.Errorf("refused: %d files were not read whole, so what they name is not known; nothing was "+ "deleted (first: %s)", len(s.NotRead), s.NotRead[0]) } del, cancel := context.WithTimeout(context.Background(), budget) defer cancel() for _, c := range p.Candidates { if err := f.Delete(del, c.Package, c.Version); err != nil && !errors.Is(err, errNotFound) { p.Refused = append(p.Refused, fmt.Sprintf("%s@%s: %v", c.Package, c.Version, err)) continue } p.Deleted = append(p.Deleted, c.Package+"@"+c.Version) } s.Said = append(s.Said, fmt.Sprintf("deleted %d, refused %d, because: %s", len(p.Deleted), len(p.Refused), p.Why)) return p, nil } func countVersions(s *Survey) int { n := 0 for _, p := range s.Packages { n += len(p.Versions) } return n } // sizesSaid says, when any version's size could not be read, how many and the first reason: those versions // count 0 bytes, so every total is at least, not exactly. func sizesSaid(s *Survey) []string { var failed []string for _, p := range s.Packages { for _, v := range p.Versions { if v.SizeError != "" { failed = append(failed, fmt.Sprintf("%s@%s: %s", p.Name, v.Version, v.SizeError)) } } } if len(failed) == 0 { return nil } return []string{fmt.Sprintf("the size of %d versions could not be read, so the bytes said are at least, not exactly "+ "(first: %s)", len(failed), failed[0])} } func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }