mesh-catalog/modules/gitea/cmd/npm-registry/retention.go

package main

import (
	"context"
	"errors"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"
)

// Retention for the package registry (novox/hq ADR 0251 §4, to-be 51): a version is kept when a
// lockfile on a repository's default branch names it, when it is the highest version satisfying a
// range a package manifest there names, when a dist-tag names it, or when it is among the newest
// `keep` of its package. Everything else is what retention would delete.

// DefaultKeep is how many of each package's newest versions are kept whatever names them: the same
// five builds the artifact store keeps (ADR 0189 §3), so "how far back" has one answer.
const DefaultKeep = 5

// workers bounds how many calls go to the forge at once; budget how long one tool call may read.
const (
	workers = 8
	budget  = 50 * time.Second
)

// Item is one version, with what keeps it.
type Item struct {
	Version   string    `json:"version"`
	Published time.Time `json:"published"`
	Bytes     int64     `json:"bytes"`
	SizeError string    `json:"size_error,omitempty"`
	Kept      bool      `json:"kept"`
	Why       []string  `json:"why,omitempty"`
}

// Package is one package's versions, newest first.
type Package struct {
	Name     string            `json:"name"`
	DistTags map[string]string `json:"dist_tags,omitempty"`
	// KeepsAll says why every version is kept, when something about the package could not be read.
	KeepsAll string `json:"keeps_all,omitempty"`
	Versions []Item `json:"versions"`
}

// Survey is the registry and what the forge's repositories say of it.
type Survey struct {
	Owner        string    `json:"owner"`
	Keep         int       `json:"keep"`
	Repositories int       `json:"repositories_read"`
	Unread       []string  `json:"repositories_unread,omitempty"`
	NotRead      []string  `json:"files_not_read,omitempty"`
	Packages     []Package `json:"packages"`
	Said         []string  `json:"said"`
}

// survey reads the registry and every repository, and decides what is kept. sizes says whether to
// ask the forge each version's size (for these versions only, when not nil).
func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Item) bool) (*Survey, error) {
	ctx, cancel := context.WithTimeout(ctx, budget)
	defer cancel()
	listed, err := f.Packages(ctx)
	if err != nil {
		return nil, err
	}
	byName := map[string][]PackageVersion{}
	for _, v := range listed {
		byName[v.Name] = append(byName[v.Name], v)
	}
	if only != "" {
		if _, has := byName[only]; !has {
			return nil, fmt.Errorf("the registry of %s holds no npm package %q", f.Owner, only)
		}
	}
	known := map[string]bool{}
	for name := range byName {
		known[name] = true
	}

	s := &Survey{Owner: f.Owner, Keep: keep, Said: []string{}}
	wanted, pinned, err := readRepositories(ctx, f, known, s)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(byName))
	for name := range byName {
		if only == "" || name == only {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	tags := distTags(ctx, f, names)
	for _, name := range names {
		s.Packages = append(s.Packages, decide(name, byName[name], tags[name], wanted[name], pinned[name], keep))
	}
	if sizes != nil {
		measure(ctx, f, s, sizes)
	}
	return s, nil
}

type tagAnswer struct {
	tags map[string]string
	err  error
}

func distTags(ctx context.Context, f *Forge, names []string) map[string]tagAnswer {
	out := map[string]tagAnswer{}
	var mu sync.Mutex
	each(names, func(name string) {
		t, err := f.DistTags(ctx, name)
		mu.Lock()
		out[name] = tagAnswer{t, err}
		mu.Unlock()
	})
	return out
}

// each runs fn over items, workers at a time.
func each[T any](items []T, fn func(T)) {
	var wg sync.WaitGroup
	gate := make(chan struct{}, workers)
	for _, item := range items {
		wg.Add(1)
		gate <- struct{}{}
		go func(item T) {
			defer wg.Done()
			defer func() { <-gate }()
			fn(item)
		}(item)
	}
	wg.Wait()
}

// readRepositories reads every repository's manifests and lockfiles. A repository that could not be
// read is named in Unread: what it depends on is not known, so nothing may be deleted.
func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *Survey) (map[string][]Wanted, map[string][]Pinned, error) {
	repos, err := f.Repositories(ctx)
	if err != nil {
		return nil, nil, fmt.Errorf("the forge's repositories cannot be listed, so nothing is known to depend on anything: %w", err)
	}
	wanted := map[string][]Wanted{}
	pinned := map[string][]Pinned{}
	var mu sync.Mutex
	each(repos, func(r Repository) {
		if r.Empty || r.DefaultBranch == "" {
			mu.Lock()
			s.Repositories++
			mu.Unlock()
			return
		}
		files, err := f.Files(ctx, r)
		if errors.Is(err, errNotFound) {
			// A repository without its default branch holds nothing to read.
			files, err = nil, nil
		}
		if err != nil {
			mu.Lock()
			s.Unread = append(s.Unread, fmt.Sprintf("%s: %v", r.FullName, err))
			mu.Unlock()
			return
		}
		var ws []Wanted
		var ps []Pinned
		var notRead, unread []string
		for _, p := range files {
			if !interesting(p) {
				continue
			}
			where := r.FullName + ":" + p
			content, err := f.Raw(ctx, r, p)
			if err != nil {
				unread = append(unread, fmt.Sprintf("%s: %v", where, err))
				continue
			}
			base := path.Base(p)
			if manifestNames[base] {
				got, err := ReadManifest(content, where, known)
				if err != nil {
					notRead = append(notRead, fmt.Sprintf("%s: not JSON (%v)", where, err))
					continue
				}
				ws = append(ws, got...)
				continue
			}
			got, err := ReadLock(base, content, where, known)
			if err != nil {
				notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err))
				continue
			}
			ps = append(ps, got...)
		}
		mu.Lock()
		defer mu.Unlock()
		s.Repositories++
		s.Unread = append(s.Unread, unread...)
		s.NotRead = append(s.NotRead, notRead...)
		for _, w := range ws {
			wanted[w.Package] = append(wanted[w.Package], w)
		}
		for _, p := range ps {
			pinned[p.Package] = append(pinned[p.Package], p)
		}
	})
	sort.Strings(s.Unread)
	sort.Strings(s.NotRead)
	return wanted, pinned, nil
}

// decide is one package's versions, newest first, each with what keeps it.
func decide(name string, listed []PackageVersion, tags tagAnswer, wanted []Wanted, pinned []Pinned, keep int) Package {
	p := Package{Name: name, DistTags: tags.tags}
	type entry struct {
		item Item
		v    Version
		ok   bool
	}
	var entries []*entry
	byVersion := map[string]*entry{}
	var parsed []Version
	for _, l := range listed {
		e := &entry{item: Item{Version: l.Version, Published: l.CreatedAt}}
		e.v, e.ok = mustVersion(l.Version)
		entries = append(entries, e)
		byVersion[l.Version] = e
		if e.ok {
			parsed = append(parsed, e.v)
		}
	}
	why := func(version, reason string) {
		if e := byVersion[version]; e != nil {
			e.item.Why = append(e.item.Why, reason)
		}
	}

	var keepsAll []string
	if tags.err != nil {
		keepsAll = append(keepsAll, fmt.Sprintf("its dist-tags could not be read (%v)", tags.err))
	}
	for tag, version := range tags.tags {
		why(version, "dist-tag "+tag)
	}
	for _, pin := range pinned {
		why(pin.Version, "a lockfile names it: "+pin.Where)
	}
	for _, w := range wanted {
		if version, isTag := tags.tags[w.Range]; isTag {
			why(version, fmt.Sprintf("dist-tag %q is the range %s names", w.Range, w.Where))
			continue
		}
		r, err := ParseRange(w.Range)
		if err != nil {
			keepsAll = append(keepsAll, fmt.Sprintf("the range %q in %s cannot be read (%v)", w.Range, w.Where, err))
			continue
		}
		if best, found := MaxSatisfying(r, parsed); found {
			why(best.String(), fmt.Sprintf("the highest version satisfying %s in %s", w.Range, w.Where))
		}
	}

	sort.SliceStable(entries, func(i, j int) bool {
		a, b := entries[i], entries[j]
		switch {
		case a.ok && b.ok:
			return Compare(a.v, b.v) > 0
		case a.ok != b.ok:
			return a.ok // a version that is not semver sorts last
		}
		return a.item.Published.After(b.item.Published)
	})
	newest := 0
	for _, e := range entries {
		if !e.ok {
			e.item.Why = append(e.item.Why, "not a semantic version, so not ordered: kept")
			continue
		}
		if newest < keep {
			newest++
			e.item.Why = append(e.item.Why, fmt.Sprintf("among the newest %d (pre-releases count)", keep))
		}
	}
	if len(keepsAll) > 0 {
		p.KeepsAll = strings.Join(keepsAll, "; ")
	}
	for _, e := range entries {
		e.item.Why = dedupe(e.item.Why)
		e.item.Kept = len(e.item.Why) > 0 || p.KeepsAll != ""
		p.Versions = append(p.Versions, e.item)
	}
	return p
}

func mustVersion(s string) (Version, bool) {
	v, err := ParseVersion(s)
	return v, err == nil
}

// dedupe keeps each reason once and at most a few of each kind, so a version a hundred lockfiles name
// is not a hundred lines.
func dedupe(why []string) []string {
	seen := map[string]bool{}
	kinds := map[string]int{}
	var out []string
	more := map[string]int{}
	for _, w := range why {
		if seen[w] {
			continue
		}
		seen[w] = true
		kind, _, _ := strings.Cut(w, ":")
		if kinds[kind] >= 3 {
			more[kind]++
			continue
		}
		kinds[kind]++
		out = append(out, w)
	}
	for kind, n := range more {
		out = append(out, fmt.Sprintf("%s: and %d more", kind, n))
	}
	sort.Strings(out)
	return out
}

// measure asks the forge the size of each version sizes picks.
func measure(ctx context.Context, f *Forge, s *Survey, sizes func(Item) bool) {
	type at struct{ p, v int }
	var todo []at
	for i := range s.Packages {
		for j := range s.Packages[i].Versions {
			if sizes(s.Packages[i].Versions[j]) {
				todo = append(todo, at{i, j})
			}
		}
	}
	var mu sync.Mutex
	each(todo, func(a at) {
		p := &s.Packages[a.p]
		n, err := f.Size(ctx, p.Name, p.Versions[a.v].Version)
		mu.Lock()
		defer mu.Unlock()
		if err != nil {
			p.Versions[a.v].SizeError = err.Error()
			return
		}
		p.Versions[a.v].Bytes = n
	})
}

// Candidate is one version retention would delete.
type Candidate struct {
	Package   string    `json:"package"`
	Version   string    `json:"version"`
	Published time.Time `json:"published"`
	Bytes     int64     `json:"bytes"`
}

// Plan is what retention would delete, and — for a real run — what it did.
type Plan struct {
	*Survey
	DryRun     bool        `json:"dry_run"`
	Why        string      `json:"why,omitempty"`
	Candidates []Candidate `json:"would_delete"`
	Bytes      int64       `json:"bytes"`
	Deleted    []string    `json:"deleted,omitempty"`
	Refused    []string    `json:"refused,omitempty"`
}

// retention works out the plan, and carries it out when dryRun is false.
func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool, why string) (*Plan, error) {
	if !dryRun && strings.TrimSpace(why) == "" {
		return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done")
	}
	s, err := survey(ctx, f, only, keep, func(i Item) bool { return !i.Kept })
	if err != nil {
		return nil, err
	}
	p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}}
	for _, pkg := range s.Packages {
		for _, v := range pkg.Versions {
			if !v.Kept {
				p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes})
				p.Bytes += v.Bytes
			}
		}
	}
	s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages; %d would be deleted, %s",
		s.Repositories, countVersions(s), len(s.Packages), len(p.Candidates), mib(p.Bytes)))
	if len(s.Unread) > 0 {
		s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+
			"known, so a real run deletes nothing", len(s.Unread)))
	}
	if dryRun {
		s.Said = append(s.Said, "a dry run: nothing was deleted")
		return p, nil
	}
	if len(s.Unread) > 0 {
		return p, fmt.Errorf("refused: %d repositories or files could not be read, so what depends on what is not "+
			"known; nothing was deleted (first: %s)", len(s.Unread), s.Unread[0])
	}
	del, cancel := context.WithTimeout(context.Background(), budget)
	defer cancel()
	for _, c := range p.Candidates {
		if err := f.Delete(del, c.Package, c.Version); err != nil && !errors.Is(err, errNotFound) {
			p.Refused = append(p.Refused, fmt.Sprintf("%s@%s: %v", c.Package, c.Version, err))
			continue
		}
		p.Deleted = append(p.Deleted, c.Package+"@"+c.Version)
	}
	s.Said = append(s.Said, fmt.Sprintf("deleted %d, refused %d, because: %s", len(p.Deleted), len(p.Refused), p.Why))
	return p, nil
}

func countVersions(s *Survey) int {
	n := 0
	for _, p := range s.Packages {
		n += len(p.Versions)
	}
	return n
}

func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }