Merge pull request 'gitea: read npm lockfiles of every version and measure every version's size' (#128) from fix/npm-lockfile-shapes into main
This commit was merged in pull request #128.
This commit is contained in:
@@ -35,8 +35,9 @@ why under `keeps_all`.
|
||||
**Nothing is deleted on partial knowledge.** A repository whose files could not be listed, or a
|
||||
manifest or lockfile that could not be fetched, is named under `repositories_unread` in every answer,
|
||||
and a real run then deletes nothing at all. A file that was fetched but is not readable (broken JSON) is
|
||||
named under `files_not_read`; what it would have named is not known, and the operator reads that list
|
||||
before a real run. Under `node_modules` nothing is read: that is what a lockfile already says. An empty
|
||||
named under `files_not_read`, and so is a lockfile entry of a shape the reader does not know (the rest of
|
||||
that file is still read); what it would have named is not known, so a real run then deletes nothing at all
|
||||
either. Under `node_modules` nothing is read: that is what a lockfile already says. An empty
|
||||
repository, or one without its default branch, holds nothing to read and is not a failure.
|
||||
|
||||
A real run deletes each version through the forge's own interface (`DELETE
|
||||
|
||||
@@ -78,6 +78,9 @@ func (f *Forge) password() (string, error) {
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// maxBody is the largest answer read from the forge.
|
||||
const maxBody = 32 << 20
|
||||
|
||||
// do sends one request and answers the body; 404 is errNotFound.
|
||||
func (f *Forge) do(ctx context.Context, method, path string) ([]byte, error) {
|
||||
password, err := f.password()
|
||||
@@ -95,10 +98,15 @@ func (f *Forge) do(ctx context.Context, method, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, 32<<20))
|
||||
// One byte past the cap is read, so a body the cap cut is told from one that fits it exactly: a
|
||||
// cut body is refused, never parsed as if it were whole.
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxBody+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(body) > maxBody {
|
||||
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, maxBody>>20)
|
||||
}
|
||||
switch {
|
||||
case res.StatusCode == http.StatusNotFound:
|
||||
return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound)
|
||||
|
||||
@@ -65,6 +65,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
||||
}
|
||||
s.Said = append(s.Said, fmt.Sprintf("%d packages, %d versions, %d kept; %d repositories read, %d unread",
|
||||
len(s.Packages), total, kept, s.Repositories, len(s.Unread)))
|
||||
s.Said = append(s.Said, sizesSaid(s)...)
|
||||
return s, nil
|
||||
},
|
||||
},
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -89,43 +91,70 @@ func notRegistry(spec string) bool {
|
||||
return strings.Contains(spec, "/") && !strings.HasPrefix(spec, "npm:")
|
||||
}
|
||||
|
||||
// ReadLock is every registry package version a lockfile names.
|
||||
// ReadLock is every registry package version a lockfile names. An error beside pins says what part of the
|
||||
// file was not read; the pins are what was.
|
||||
func ReadLock(base string, content []byte, where string, known map[string]bool) ([]Pinned, error) {
|
||||
switch base {
|
||||
case "package-lock.json", "npm-shrinkwrap.json":
|
||||
return readNpmLock(content, where, known)
|
||||
case "yarn.lock":
|
||||
return readYarnLock(content, where, known), nil
|
||||
return readYarnLock(content, where, known)
|
||||
case "pnpm-lock.yaml":
|
||||
return readTextLock(content, where, known), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type npmLockEntry struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Dependencies map[string]npmLockEntry `json:"dependencies"`
|
||||
// npmPackage is one entry of a version 2 or 3 lockfile's `packages`, keyed by the path it is installed
|
||||
// at. Only what says which version it is is read; every other field (`dependencies` there is a map of
|
||||
// name to range, `requires`, `engines`, `bin`, ...) is left alone, so a field this does not know cannot
|
||||
// make the file unreadable.
|
||||
type npmPackage struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Link bool `json:"link"`
|
||||
Resolved string `json:"resolved"`
|
||||
}
|
||||
|
||||
// npmDependency is one entry of a version 1 (and 2's copy) lockfile's `dependencies`: nested by name,
|
||||
// with its own `dependencies` of the same shape. `requires` is a map of name to range, and not read.
|
||||
type npmDependency struct {
|
||||
Version string `json:"version"`
|
||||
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
||||
}
|
||||
|
||||
// readNpmLock reads package-lock.json and npm-shrinkwrap.json of every lockfileVersion: 1 (nested
|
||||
// `dependencies`), 2 (both) and 3 (`packages` only). It is read entry by entry, so one entry of a shape
|
||||
// it does not know is said and skipped, never the whole file: the pins it did read are answered with an
|
||||
// error naming what was not.
|
||||
func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, error) {
|
||||
var lock struct {
|
||||
Packages map[string]npmLockEntry `json:"packages"`
|
||||
Dependencies map[string]npmLockEntry `json:"dependencies"`
|
||||
Packages map[string]json.RawMessage `json:"packages"`
|
||||
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &lock); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Pinned
|
||||
var skipped []string
|
||||
add := func(name, version string) {
|
||||
if known[name] && version != "" {
|
||||
out = append(out, Pinned{Package: name, Version: version, Where: where})
|
||||
}
|
||||
}
|
||||
// Version 2 and 3: keyed by the path it is installed at. An alias carries the real name.
|
||||
for key, e := range lock.Packages {
|
||||
// Version 2 and 3: keyed by the path it is installed at; the name is what follows the last
|
||||
// node_modules/. An alias carries the real name. A link is a workspace on disk, not a registry version.
|
||||
for key, raw := range lock.Packages {
|
||||
_, name, found := cutLast(key, "node_modules/")
|
||||
if !found {
|
||||
continue // the root, or a workspace's own directory
|
||||
}
|
||||
var e npmPackage
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
skipped = append(skipped, fmt.Sprintf("packages[%q] (%v)", key, err))
|
||||
continue
|
||||
}
|
||||
if e.Link {
|
||||
continue
|
||||
}
|
||||
if e.Name != "" {
|
||||
@@ -134,9 +163,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned,
|
||||
add(name, e.Version)
|
||||
}
|
||||
// Version 1 (and 2's copy): nested by name. An alias's version is `npm:<name>@<version>`.
|
||||
var walk func(map[string]npmLockEntry)
|
||||
walk = func(deps map[string]npmLockEntry) {
|
||||
for name, e := range deps {
|
||||
var walk func(trail string, deps map[string]json.RawMessage)
|
||||
walk = func(trail string, deps map[string]json.RawMessage) {
|
||||
for name, raw := range deps {
|
||||
var e npmDependency
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
skipped = append(skipped, fmt.Sprintf("dependencies%s[%q] (%v)", trail, name, err))
|
||||
continue
|
||||
}
|
||||
if alias, ok := strings.CutPrefix(e.Version, "npm:"); ok {
|
||||
if at := strings.LastIndex(alias, "@"); at > 0 {
|
||||
add(alias[:at], alias[at+1:])
|
||||
@@ -144,10 +178,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned,
|
||||
} else {
|
||||
add(name, e.Version)
|
||||
}
|
||||
walk(e.Dependencies)
|
||||
walk(fmt.Sprintf("%s[%q].dependencies", trail, name), e.Dependencies)
|
||||
}
|
||||
}
|
||||
walk(lock.Dependencies)
|
||||
walk("", lock.Dependencies)
|
||||
if len(skipped) > 0 {
|
||||
sort.Strings(skipped)
|
||||
return out, fmt.Errorf("%d entries not read, the rest were: %s", len(skipped), strings.Join(skipped, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -161,7 +199,10 @@ func cutLast(s, sep string) (string, string, bool) {
|
||||
|
||||
// readYarnLock reads both yarn formats: a header of the specs it resolves (`"@a/b@^1.0.0", "@a/b@^1.1.0":`)
|
||||
// and an indented `version "1.2.3"` (classic) or `version: 1.2.3` (berry) under it.
|
||||
func readYarnLock(content []byte, where string, known map[string]bool) []Pinned {
|
||||
//
|
||||
// A line longer than the scanner's buffer stops it; that is an error, never a lockfile read to its end,
|
||||
// because what follows the line pins versions too.
|
||||
func readYarnLock(content []byte, where string, known map[string]bool) ([]Pinned, error) {
|
||||
var out []Pinned
|
||||
var names []string
|
||||
scanner := bufio.NewScanner(bytes.NewReader(content))
|
||||
@@ -191,7 +232,10 @@ func readYarnLock(content []byte, where string, known map[string]bool) []Pinned
|
||||
addYarn(&out, names, strings.Trim(v, `"`), where, known)
|
||||
}
|
||||
}
|
||||
return out
|
||||
if err := scanner.Err(); err != nil {
|
||||
return out, fmt.Errorf("%s was not read to its end: %w", where, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func addYarn(out *[]Pinned, names []string, version, where string, known map[string]bool) {
|
||||
|
||||
@@ -66,11 +66,11 @@ func TestAnNpmLockfileOfEveryVersionNamesWhatItPins(t *testing.T) {
|
||||
|
||||
func TestAYarnLockOfEitherFormatNamesWhatItPins(t *testing.T) {
|
||||
classic := "# yarn lockfile v1\n\n\"@novox/mesh-sdk@^0.1.0\", \"@novox/mesh-sdk@^0.1.2\":\n version \"0.1.4\"\n resolved \"x\"\n dependencies:\n left-pad \"^1\"\n\nleft-pad@^1:\n version \"1.3.0\"\n"
|
||||
if got := pins(readYarnLock([]byte(classic), "r", knownHere)); got != "@novox/mesh-sdk@0.1.4" {
|
||||
if got := pins(mustYarn(t, classic)); got != "@novox/mesh-sdk@0.1.4" {
|
||||
t.Fatalf("classic: %s", got)
|
||||
}
|
||||
berry := "__metadata:\n version: 6\n\n\"@novox/ui@npm:^2.0.0\":\n version: 2.0.7\n resolution: \"@novox/ui@npm:2.0.7\"\n"
|
||||
if got := pins(readYarnLock([]byte(berry), "r", knownHere)); got != "@novox/ui@2.0.7" {
|
||||
if got := pins(mustYarn(t, berry)); got != "@novox/ui@2.0.7" {
|
||||
t.Fatalf("berry: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -90,3 +90,126 @@ func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Lockfiles as npm writes them: a `packages` entry's `dependencies` is a map of name to range, which a
|
||||
// reader expecting nested entries cannot read (the shape every lockfile on the forge had).
|
||||
|
||||
const lockV1 = `{
|
||||
"name": "app", "version": "1.0.0", "lockfileVersion": 1, "requires": true,
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://forge.example/api/packages/novox/npm/@novox/mesh-sdk/-/0.1.2/mesh-sdk-0.1.2.tgz",
|
||||
"integrity": "sha512-x",
|
||||
"requires": {"@novox/ui": "^1.0.0", "left-pad": "^1.3.0"},
|
||||
"dependencies": {
|
||||
"@novox/ui": {"version": "1.0.0", "resolved": "https://forge.example/x.tgz", "integrity": "sha512-y",
|
||||
"requires": {"left-pad": "^1.0.0"},
|
||||
"dependencies": {"@novox/mesh-sdk": {"version": "0.1.0", "requires": {}}}}
|
||||
}
|
||||
},
|
||||
"left-pad": {"version": "1.3.0", "dev": true},
|
||||
"sdk": {"version": "npm:@novox/ui@2.0.1"},
|
||||
"local": {"version": "file:../local", "requires": {"@novox/ui": "*"}}
|
||||
}
|
||||
}`
|
||||
|
||||
const lockV2 = `{
|
||||
"name": "app", "version": "1.0.0", "lockfileVersion": 2, "requires": true,
|
||||
"packages": {
|
||||
"": {"name": "app", "version": "1.0.0", "dependencies": {"@novox/mesh-sdk": "^0.1.0"},
|
||||
"devDependencies": {"@novox/ui": "~1.1.0"}, "engines": {"node": ">=20"}},
|
||||
"node_modules/@novox/mesh-sdk": {"version": "0.1.3", "resolved": "https://forge.example/a.tgz",
|
||||
"integrity": "sha512-a", "dependencies": {"left-pad": "^1.3.0"}, "peerDependencies": {"@novox/ui": "*"},
|
||||
"optionalDependencies": {"fsevents": "^2"}, "bin": {"sdk": "bin/sdk.js"}, "engines": {"node": ">=18"}},
|
||||
"node_modules/@novox/ui": {"version": "1.1.4", "dev": true, "funding": [{"type": "x", "url": "y"}]},
|
||||
"node_modules/left-pad": {"version": "1.3.0"}
|
||||
},
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": {"version": "0.1.3", "requires": {"left-pad": "^1.3.0"}},
|
||||
"@novox/ui": {"version": "1.1.4", "dev": true},
|
||||
"left-pad": {"version": "1.3.0"}
|
||||
}
|
||||
}`
|
||||
|
||||
const lockV3 = `{
|
||||
"name": "app", "version": "1.0.0", "lockfileVersion": 3, "requires": true,
|
||||
"packages": {
|
||||
"": {"name": "app", "workspaces": ["packages/*"], "dependencies": {"@novox/mesh-sdk": "^0.1.0", "sdk": "npm:@novox/ui@^2.0.0"}},
|
||||
"node_modules/@novox/mesh-sdk": {"version": "0.1.5", "license": "MIT", "dependencies": {"@novox/ui": "^1.0.0"}},
|
||||
"node_modules/@novox/ui": {"version": "1.0.9", "devDependencies": {"typescript": "^5"}},
|
||||
"node_modules/a/node_modules/@novox/mesh-sdk": {"version": "0.1.1", "dependencies": {"left-pad": "1.x"}},
|
||||
"node_modules/a/node_modules/b/node_modules/@novox/ui": {"version": "1.0.2"},
|
||||
"node_modules/sdk": {"name": "@novox/ui", "version": "2.0.4", "resolved": "https://forge.example/u.tgz"},
|
||||
"node_modules/@novox/workspace-thing": {"resolved": "packages/thing", "link": true},
|
||||
"node_modules/@novox/linked": {"resolved": "packages/linked", "link": true, "version": "9.9.9"},
|
||||
"packages/thing": {"name": "@novox/workspace-thing", "version": "0.0.1", "dependencies": {"@novox/ui": "^1.0.0"}},
|
||||
"node_modules/left-pad": {"version": "1.3.0", "peerDependencies": {"x": "*"}, "optionalDependencies": {"y": "1"}}
|
||||
}
|
||||
}`
|
||||
|
||||
var knownLocks = map[string]bool{"@novox/mesh-sdk": true, "@novox/ui": true, "@novox/workspace-thing": true, "@novox/linked": true}
|
||||
|
||||
func TestALockfileVersion1NamesEveryNestedVersion(t *testing.T) {
|
||||
got, err := ReadLock("package-lock.json", []byte(lockV1), "r", knownLocks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pins(got) != "@novox/mesh-sdk@0.1.0 @novox/mesh-sdk@0.1.2 @novox/ui@1.0.0 @novox/ui@2.0.1" {
|
||||
t.Fatalf("v1: %s", pins(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockfileVersion2ReadsBothHalves(t *testing.T) {
|
||||
got, err := ReadLock("package-lock.json", []byte(lockV2), "r", knownLocks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Each half names both; a version is kept once it is named, so naming it twice is harmless.
|
||||
if pins(got) != "@novox/mesh-sdk@0.1.3 @novox/mesh-sdk@0.1.3 @novox/ui@1.1.4 @novox/ui@1.1.4" {
|
||||
t.Fatalf("v2: %s", pins(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockfileVersion3ReadsPathsAliasesAndSkipsLinks(t *testing.T) {
|
||||
got, err := ReadLock("package-lock.json", []byte(lockV3), "r", knownLocks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "@novox/mesh-sdk@0.1.1 @novox/mesh-sdk@0.1.5 @novox/ui@1.0.2 @novox/ui@1.0.9 @novox/ui@2.0.4"
|
||||
if pins(got) != want {
|
||||
t.Fatalf("v3: %s\nwant %s", pins(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEntryOfAnUnknownShapeIsSaidAndTheRestIsRead(t *testing.T) {
|
||||
lock := `{"lockfileVersion":3,"packages":{
|
||||
"node_modules/@novox/ui":{"version":"1.0.0"},
|
||||
"node_modules/@novox/mesh-sdk":{"version":{"odd":true}}},
|
||||
"dependencies":{"@novox/ui":"not an object"}}`
|
||||
got, err := ReadLock("package-lock.json", []byte(lock), "r", knownLocks)
|
||||
if pins(got) != "@novox/ui@1.0.0" {
|
||||
t.Fatalf("read %s", pins(got))
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "2 entries not read") ||
|
||||
!strings.Contains(err.Error(), `packages["node_modules/@novox/mesh-sdk"]`) {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustYarn(t *testing.T, content string) []Pinned {
|
||||
t.Helper()
|
||||
out, err := readYarnLock([]byte(content), "r", knownHere)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A yarn lockfile with a line longer than the scanner holds is an error, not a file read to its end.
|
||||
func TestAYarnLockfileCutShortIsAnError(t *testing.T) {
|
||||
long := "\"@novox/mesh-sdk@^0.1.0\":\n version \"0.1.4\"\n# " + strings.Repeat("x", 2<<20) + "\n\"@novox/ui@^2\":\n version \"2.0.7\"\n"
|
||||
if _, err := readYarnLock([]byte(long), "r", knownHere); err == nil {
|
||||
t.Fatal("a lockfile the scanner stopped in was read as whole")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,11 +187,10 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S
|
||||
continue
|
||||
}
|
||||
got, err := ReadLock(base, content, where, known)
|
||||
ps = append(ps, got...)
|
||||
if err != nil {
|
||||
notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err))
|
||||
continue
|
||||
}
|
||||
ps = append(ps, got...)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
@@ -356,6 +355,7 @@ type Candidate struct {
|
||||
Version string `json:"version"`
|
||||
Published time.Time `json:"published"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SizeError string `json:"size_error,omitempty"`
|
||||
}
|
||||
|
||||
// Plan is what retention would delete, and — for a real run — what it did.
|
||||
@@ -365,8 +365,10 @@ type Plan struct {
|
||||
Why string `json:"why,omitempty"`
|
||||
Candidates []Candidate `json:"would_delete"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Deleted []string `json:"deleted,omitempty"`
|
||||
Refused []string `json:"refused,omitempty"`
|
||||
// RegistryBytes is every version's size together, deleted or kept: what retention is measured against.
|
||||
RegistryBytes int64 `json:"registry_bytes"`
|
||||
Deleted []string `json:"deleted,omitempty"`
|
||||
Refused []string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
// retention works out the plan, and carries it out when dryRun is false.
|
||||
@@ -374,21 +376,28 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool
|
||||
if !dryRun && strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done")
|
||||
}
|
||||
s, err := survey(ctx, f, only, keep, func(i Item) bool { return !i.Kept })
|
||||
// Every version is measured, kept or not, so no size in the answer is a 0 nobody asked for.
|
||||
s, err := survey(ctx, f, only, keep, func(Item) bool { return true })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}}
|
||||
for _, pkg := range s.Packages {
|
||||
for _, v := range pkg.Versions {
|
||||
p.RegistryBytes += v.Bytes
|
||||
if !v.Kept {
|
||||
p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes})
|
||||
p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes, v.SizeError})
|
||||
p.Bytes += v.Bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages; %d would be deleted, %s",
|
||||
s.Repositories, countVersions(s), len(s.Packages), len(p.Candidates), mib(p.Bytes)))
|
||||
s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages, %s; %d would be deleted, %s",
|
||||
s.Repositories, countVersions(s), len(s.Packages), mib(p.RegistryBytes), len(p.Candidates), mib(p.Bytes)))
|
||||
s.Said = append(s.Said, sizesSaid(s)...)
|
||||
if len(s.NotRead) > 0 {
|
||||
s.Said = append(s.Said, fmt.Sprintf("%d files were not read whole (files_not_read): what they name is not "+
|
||||
"known, so a real run deletes nothing", len(s.NotRead)))
|
||||
}
|
||||
if len(s.Unread) > 0 {
|
||||
s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+
|
||||
"known, so a real run deletes nothing", len(s.Unread)))
|
||||
@@ -401,6 +410,12 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool
|
||||
return p, fmt.Errorf("refused: %d repositories or files could not be read, so what depends on what is not "+
|
||||
"known; nothing was deleted (first: %s)", len(s.Unread), s.Unread[0])
|
||||
}
|
||||
// A file that was fetched but not read whole is a file of its repository that could not be read
|
||||
// (novox/hq ADR 0251 §4): a version only it names would be deleted while in use.
|
||||
if len(s.NotRead) > 0 {
|
||||
return p, fmt.Errorf("refused: %d files were not read whole, so what they name is not known; nothing was "+
|
||||
"deleted (first: %s)", len(s.NotRead), s.NotRead[0])
|
||||
}
|
||||
del, cancel := context.WithTimeout(context.Background(), budget)
|
||||
defer cancel()
|
||||
for _, c := range p.Candidates {
|
||||
@@ -422,4 +437,22 @@ func countVersions(s *Survey) int {
|
||||
return n
|
||||
}
|
||||
|
||||
// sizesSaid says, when any version's size could not be read, how many and the first reason: those versions
|
||||
// count 0 bytes, so every total is at least, not exactly.
|
||||
func sizesSaid(s *Survey) []string {
|
||||
var failed []string
|
||||
for _, p := range s.Packages {
|
||||
for _, v := range p.Versions {
|
||||
if v.SizeError != "" {
|
||||
failed = append(failed, fmt.Sprintf("%s@%s: %s", p.Name, v.Version, v.SizeError))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []string{fmt.Sprintf("the size of %d versions could not be read, so the bytes said are at least, not exactly "+
|
||||
"(first: %s)", len(failed), failed[0])}
|
||||
}
|
||||
|
||||
func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }
|
||||
|
||||
@@ -24,6 +24,8 @@ type fakeForge struct {
|
||||
tags map[string]map[string]string // package -> dist-tags
|
||||
repos map[string]map[string]string // repository -> path -> content
|
||||
broken map[string]bool // repositories whose tree answers 500
|
||||
files map[string][]int64 // package@version -> its files' sizes (default 1 MiB + 1 KiB)
|
||||
noSize map[string]bool // package@version whose files answer 500
|
||||
mu sync.Mutex
|
||||
deleted []string
|
||||
password string
|
||||
@@ -74,7 +76,19 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
|
||||
name, _ := url.PathUnescape(m[1])
|
||||
version, _ := url.PathUnescape(m[2])
|
||||
if m[3] != "" {
|
||||
writeJSON(w, []map[string]any{{"name": "a.tgz", "size": 1 << 20}, {"name": "b", "size": 1024}})
|
||||
if f.noSize[name+"@"+version] {
|
||||
http.Error(w, "storage unavailable", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
sizes, has := f.files[name+"@"+version]
|
||||
if !has {
|
||||
sizes = []int64{1 << 20, 1024}
|
||||
}
|
||||
var files []map[string]any
|
||||
for i, n := range sizes {
|
||||
files = append(files, map[string]any{"id": i + 1, "size": n, "name": fmt.Sprintf("f%d", i), "md5": "x"})
|
||||
}
|
||||
writeJSON(w, files)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
@@ -391,3 +405,108 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) {
|
||||
t.Error("ran without a forge")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryVersionIsMeasuredAndASizeNotReadIsSaid(t *testing.T) {
|
||||
f := aRegistry(t)
|
||||
f.files = map[string][]int64{"@acme/sdk@0.1.1": {1000, 234}, "@acme/sdk@0.3.0": {5000}}
|
||||
f.noSize = map[string]bool{"@acme/sdk@0.2.0": true}
|
||||
p, err := retention(context.Background(), f.serve(), "@acme/sdk", 3, true, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := decided(p.Survey)
|
||||
// A kept version is measured too: no 0 that nobody asked for.
|
||||
if got := d["@acme/sdk@0.3.0"]; !got.Kept || got.Bytes != 5000 || got.SizeError != "" {
|
||||
t.Errorf("kept 0.3.0: %+v", got)
|
||||
}
|
||||
if got := d["@acme/sdk@0.1.1"]; got.Bytes != 1234 {
|
||||
t.Errorf("0.1.1: %+v", got)
|
||||
}
|
||||
broken := d["@acme/sdk@0.2.0"]
|
||||
if broken.Bytes != 0 || !strings.Contains(broken.SizeError, "500") {
|
||||
t.Errorf("0.2.0: %+v", broken)
|
||||
}
|
||||
var candidate *Candidate
|
||||
for i := range p.Candidates {
|
||||
if p.Candidates[i].Version == "0.2.0" {
|
||||
candidate = &p.Candidates[i]
|
||||
}
|
||||
}
|
||||
if candidate == nil || candidate.SizeError == "" {
|
||||
t.Errorf("the candidate does not carry its size error: %+v", p.Candidates)
|
||||
}
|
||||
// Would delete 0.1.1 (1234), 0.1.2 and 0.2.1 (1 MiB + 1 KiB each) and 0.2.0 (unknown).
|
||||
if want := int64(1234 + 2*(1<<20+1024)); p.Bytes != want {
|
||||
t.Errorf("bytes %d, want %d", p.Bytes, want)
|
||||
}
|
||||
if p.RegistryBytes <= p.Bytes {
|
||||
t.Errorf("registry bytes %d not above %d", p.RegistryBytes, p.Bytes)
|
||||
}
|
||||
said := strings.Join(p.Said, "\n")
|
||||
if !strings.Contains(said, "the size of 1 versions could not be read") || !strings.Contains(said, "@acme/sdk@0.2.0") {
|
||||
t.Errorf("said %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockfileOfEveryShapeKeepsWhatItPins(t *testing.T) {
|
||||
f := aRegistry(t)
|
||||
f.repos["acme/v2"] = map[string]string{"package-lock.json": `{"lockfileVersion":2,"packages":{
|
||||
"":{"name":"v2","dependencies":{"@acme/sdk":"^0.2.0"}},
|
||||
"node_modules/@acme/sdk":{"version":"0.2.0","dependencies":{"left-pad":"^1.0.0"}}},
|
||||
"dependencies":{"@acme/sdk":{"version":"0.2.0","requires":{"left-pad":"^1.0.0"}}}}`}
|
||||
s, err := survey(context.Background(), f.serve(), "@acme/sdk", 1, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(s.NotRead) != 0 {
|
||||
t.Fatalf("not read %v", s.NotRead)
|
||||
}
|
||||
if got := decided(s)["@acme/sdk@0.2.0"]; !contains(got.Why, "a lockfile names it: acme/v2:package-lock.json") {
|
||||
t.Fatalf("0.2.0: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableLockfileStopsARealRunBeforeAnythingIsDeleted(t *testing.T) {
|
||||
f := aRegistry(t)
|
||||
f.repos["acme/odd"] = map[string]string{"package-lock.json": `{"lockfileVersion":3,"packages":{`}
|
||||
p, err := retention(context.Background(), f.serve(), "", 3, false, "tidy")
|
||||
if err == nil || !strings.Contains(err.Error(), "nothing was deleted") || !strings.Contains(err.Error(), "acme/odd:package-lock.json") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if len(f.deleted) != 0 {
|
||||
t.Fatalf("deleted %v while a lockfile was not read", f.deleted)
|
||||
}
|
||||
if p == nil || len(p.NotRead) != 1 || len(p.Deleted) != 0 {
|
||||
t.Fatalf("plan %+v", p)
|
||||
}
|
||||
// And the dry run says so, without failing, and still says what would go.
|
||||
dry, err := retention(context.Background(), f.serve(), "", 3, true, "")
|
||||
if err != nil || len(dry.NotRead) != 1 || len(dry.Candidates) == 0 {
|
||||
t.Fatalf("dry run: %v %+v", err, dry)
|
||||
}
|
||||
if !strings.Contains(strings.Join(dry.Said, "\n"), "1 files were not read whole") {
|
||||
t.Errorf("said %v", dry.Said)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer larger than the cap is refused, never parsed as if the cut body were whole; one exactly at
|
||||
// the cap is read.
|
||||
func TestAnAnswerTheCapCutIsRefused(t *testing.T) {
|
||||
size := maxBody + 1
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(make([]byte, size))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
file := filepath.Join(t.TempDir(), "admin.secret")
|
||||
if err := os.WriteFile(file, []byte("x\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &Forge{URL: srv.URL, User: "admin", PasswordFile: file, Owner: "acme"}
|
||||
if _, err := f.do(context.Background(), http.MethodGet, "/big"); err == nil || !strings.Contains(err.Error(), "not read whole") {
|
||||
t.Fatalf("a cut answer: %v", err)
|
||||
}
|
||||
size = maxBody
|
||||
if body, err := f.do(context.Background(), http.MethodGet, "/big"); err != nil || len(body) != maxBody {
|
||||
t.Fatalf("an answer at the cap: %d bytes, %v", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user