diff --git a/modules/gitea/README.md b/modules/gitea/README.md index 6ef48150..1cf89306 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -35,8 +35,9 @@ why under `keeps_all`. **Nothing is deleted on partial knowledge.** A repository whose files could not be listed, or a manifest or lockfile that could not be fetched, is named under `repositories_unread` in every answer, and a real run then deletes nothing at all. A file that was fetched but is not readable (broken JSON) is -named under `files_not_read`; what it would have named is not known, and the operator reads that list -before a real run. Under `node_modules` nothing is read: that is what a lockfile already says. An empty +named under `files_not_read`, and so is a lockfile entry of a shape the reader does not know (the rest of +that file is still read); what it would have named is not known, so a real run then deletes nothing at all +either. Under `node_modules` nothing is read: that is what a lockfile already says. An empty repository, or one without its default branch, holds nothing to read and is not a failure. A real run deletes each version through the forge's own interface (`DELETE diff --git a/modules/gitea/cmd/npm-registry/forge.go b/modules/gitea/cmd/npm-registry/forge.go index 5ce649ec..948a3d21 100644 --- a/modules/gitea/cmd/npm-registry/forge.go +++ b/modules/gitea/cmd/npm-registry/forge.go @@ -78,6 +78,9 @@ func (f *Forge) password() (string, error) { return strings.TrimSpace(string(raw)), nil } +// maxBody is the largest answer read from the forge. +const maxBody = 32 << 20 + // do sends one request and answers the body; 404 is errNotFound. func (f *Forge) do(ctx context.Context, method, path string) ([]byte, error) { password, err := f.password() @@ -95,10 +98,15 @@ func (f *Forge) do(ctx context.Context, method, path string) ([]byte, error) { return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err) } defer res.Body.Close() - body, err := io.ReadAll(io.LimitReader(res.Body, 32<<20)) + // One byte past the cap is read, so a body the cap cut is told from one that fits it exactly: a + // cut body is refused, never parsed as if it were whole. + body, err := io.ReadAll(io.LimitReader(res.Body, maxBody+1)) if err != nil { return nil, err } + if len(body) > maxBody { + return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, maxBody>>20) + } switch { case res.StatusCode == http.StatusNotFound: return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound) diff --git a/modules/gitea/cmd/npm-registry/main.go b/modules/gitea/cmd/npm-registry/main.go index 52d07f9e..e964d0c6 100644 --- a/modules/gitea/cmd/npm-registry/main.go +++ b/modules/gitea/cmd/npm-registry/main.go @@ -65,6 +65,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { } s.Said = append(s.Said, fmt.Sprintf("%d packages, %d versions, %d kept; %d repositories read, %d unread", len(s.Packages), total, kept, s.Repositories, len(s.Unread))) + s.Said = append(s.Said, sizesSaid(s)...) return s, nil }, }, diff --git a/modules/gitea/cmd/npm-registry/refs.go b/modules/gitea/cmd/npm-registry/refs.go index 18c86cda..55fcfa45 100644 --- a/modules/gitea/cmd/npm-registry/refs.go +++ b/modules/gitea/cmd/npm-registry/refs.go @@ -4,8 +4,10 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "path" "regexp" + "sort" "strings" ) @@ -89,43 +91,70 @@ func notRegistry(spec string) bool { return strings.Contains(spec, "/") && !strings.HasPrefix(spec, "npm:") } -// ReadLock is every registry package version a lockfile names. +// ReadLock is every registry package version a lockfile names. An error beside pins says what part of the +// file was not read; the pins are what was. func ReadLock(base string, content []byte, where string, known map[string]bool) ([]Pinned, error) { switch base { case "package-lock.json", "npm-shrinkwrap.json": return readNpmLock(content, where, known) case "yarn.lock": - return readYarnLock(content, where, known), nil + return readYarnLock(content, where, known) case "pnpm-lock.yaml": return readTextLock(content, where, known), nil } return nil, nil } -type npmLockEntry struct { - Name string `json:"name"` - Version string `json:"version"` - Dependencies map[string]npmLockEntry `json:"dependencies"` +// npmPackage is one entry of a version 2 or 3 lockfile's `packages`, keyed by the path it is installed +// at. Only what says which version it is is read; every other field (`dependencies` there is a map of +// name to range, `requires`, `engines`, `bin`, ...) is left alone, so a field this does not know cannot +// make the file unreadable. +type npmPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Link bool `json:"link"` + Resolved string `json:"resolved"` } +// npmDependency is one entry of a version 1 (and 2's copy) lockfile's `dependencies`: nested by name, +// with its own `dependencies` of the same shape. `requires` is a map of name to range, and not read. +type npmDependency struct { + Version string `json:"version"` + Dependencies map[string]json.RawMessage `json:"dependencies"` +} + +// readNpmLock reads package-lock.json and npm-shrinkwrap.json of every lockfileVersion: 1 (nested +// `dependencies`), 2 (both) and 3 (`packages` only). It is read entry by entry, so one entry of a shape +// it does not know is said and skipped, never the whole file: the pins it did read are answered with an +// error naming what was not. func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, error) { var lock struct { - Packages map[string]npmLockEntry `json:"packages"` - Dependencies map[string]npmLockEntry `json:"dependencies"` + Packages map[string]json.RawMessage `json:"packages"` + Dependencies map[string]json.RawMessage `json:"dependencies"` } if err := json.Unmarshal(content, &lock); err != nil { return nil, err } var out []Pinned + var skipped []string add := func(name, version string) { if known[name] && version != "" { out = append(out, Pinned{Package: name, Version: version, Where: where}) } } - // Version 2 and 3: keyed by the path it is installed at. An alias carries the real name. - for key, e := range lock.Packages { + // Version 2 and 3: keyed by the path it is installed at; the name is what follows the last + // node_modules/. An alias carries the real name. A link is a workspace on disk, not a registry version. + for key, raw := range lock.Packages { _, name, found := cutLast(key, "node_modules/") if !found { + continue // the root, or a workspace's own directory + } + var e npmPackage + if err := json.Unmarshal(raw, &e); err != nil { + skipped = append(skipped, fmt.Sprintf("packages[%q] (%v)", key, err)) + continue + } + if e.Link { continue } if e.Name != "" { @@ -134,9 +163,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, add(name, e.Version) } // Version 1 (and 2's copy): nested by name. An alias's version is `npm:@`. - var walk func(map[string]npmLockEntry) - walk = func(deps map[string]npmLockEntry) { - for name, e := range deps { + var walk func(trail string, deps map[string]json.RawMessage) + walk = func(trail string, deps map[string]json.RawMessage) { + for name, raw := range deps { + var e npmDependency + if err := json.Unmarshal(raw, &e); err != nil { + skipped = append(skipped, fmt.Sprintf("dependencies%s[%q] (%v)", trail, name, err)) + continue + } if alias, ok := strings.CutPrefix(e.Version, "npm:"); ok { if at := strings.LastIndex(alias, "@"); at > 0 { add(alias[:at], alias[at+1:]) @@ -144,10 +178,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, } else { add(name, e.Version) } - walk(e.Dependencies) + walk(fmt.Sprintf("%s[%q].dependencies", trail, name), e.Dependencies) } } - walk(lock.Dependencies) + walk("", lock.Dependencies) + if len(skipped) > 0 { + sort.Strings(skipped) + return out, fmt.Errorf("%d entries not read, the rest were: %s", len(skipped), strings.Join(skipped, "; ")) + } return out, nil } @@ -161,7 +199,10 @@ func cutLast(s, sep string) (string, string, bool) { // readYarnLock reads both yarn formats: a header of the specs it resolves (`"@a/b@^1.0.0", "@a/b@^1.1.0":`) // and an indented `version "1.2.3"` (classic) or `version: 1.2.3` (berry) under it. -func readYarnLock(content []byte, where string, known map[string]bool) []Pinned { +// +// A line longer than the scanner's buffer stops it; that is an error, never a lockfile read to its end, +// because what follows the line pins versions too. +func readYarnLock(content []byte, where string, known map[string]bool) ([]Pinned, error) { var out []Pinned var names []string scanner := bufio.NewScanner(bytes.NewReader(content)) @@ -191,7 +232,10 @@ func readYarnLock(content []byte, where string, known map[string]bool) []Pinned addYarn(&out, names, strings.Trim(v, `"`), where, known) } } - return out + if err := scanner.Err(); err != nil { + return out, fmt.Errorf("%s was not read to its end: %w", where, err) + } + return out, nil } func addYarn(out *[]Pinned, names []string, version, where string, known map[string]bool) { diff --git a/modules/gitea/cmd/npm-registry/refs_test.go b/modules/gitea/cmd/npm-registry/refs_test.go index ad75526c..430e3094 100644 --- a/modules/gitea/cmd/npm-registry/refs_test.go +++ b/modules/gitea/cmd/npm-registry/refs_test.go @@ -66,11 +66,11 @@ func TestAnNpmLockfileOfEveryVersionNamesWhatItPins(t *testing.T) { func TestAYarnLockOfEitherFormatNamesWhatItPins(t *testing.T) { classic := "# yarn lockfile v1\n\n\"@novox/mesh-sdk@^0.1.0\", \"@novox/mesh-sdk@^0.1.2\":\n version \"0.1.4\"\n resolved \"x\"\n dependencies:\n left-pad \"^1\"\n\nleft-pad@^1:\n version \"1.3.0\"\n" - if got := pins(readYarnLock([]byte(classic), "r", knownHere)); got != "@novox/mesh-sdk@0.1.4" { + if got := pins(mustYarn(t, classic)); got != "@novox/mesh-sdk@0.1.4" { t.Fatalf("classic: %s", got) } berry := "__metadata:\n version: 6\n\n\"@novox/ui@npm:^2.0.0\":\n version: 2.0.7\n resolution: \"@novox/ui@npm:2.0.7\"\n" - if got := pins(readYarnLock([]byte(berry), "r", knownHere)); got != "@novox/ui@2.0.7" { + if got := pins(mustYarn(t, berry)); got != "@novox/ui@2.0.7" { t.Fatalf("berry: %s", got) } } @@ -90,3 +90,126 @@ func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) { } } } + +// Lockfiles as npm writes them: a `packages` entry's `dependencies` is a map of name to range, which a +// reader expecting nested entries cannot read (the shape every lockfile on the forge had). + +const lockV1 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 1, "requires": true, + "dependencies": { + "@novox/mesh-sdk": { + "version": "0.1.2", + "resolved": "https://forge.example/api/packages/novox/npm/@novox/mesh-sdk/-/0.1.2/mesh-sdk-0.1.2.tgz", + "integrity": "sha512-x", + "requires": {"@novox/ui": "^1.0.0", "left-pad": "^1.3.0"}, + "dependencies": { + "@novox/ui": {"version": "1.0.0", "resolved": "https://forge.example/x.tgz", "integrity": "sha512-y", + "requires": {"left-pad": "^1.0.0"}, + "dependencies": {"@novox/mesh-sdk": {"version": "0.1.0", "requires": {}}}} + } + }, + "left-pad": {"version": "1.3.0", "dev": true}, + "sdk": {"version": "npm:@novox/ui@2.0.1"}, + "local": {"version": "file:../local", "requires": {"@novox/ui": "*"}} + } +}` + +const lockV2 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 2, "requires": true, + "packages": { + "": {"name": "app", "version": "1.0.0", "dependencies": {"@novox/mesh-sdk": "^0.1.0"}, + "devDependencies": {"@novox/ui": "~1.1.0"}, "engines": {"node": ">=20"}}, + "node_modules/@novox/mesh-sdk": {"version": "0.1.3", "resolved": "https://forge.example/a.tgz", + "integrity": "sha512-a", "dependencies": {"left-pad": "^1.3.0"}, "peerDependencies": {"@novox/ui": "*"}, + "optionalDependencies": {"fsevents": "^2"}, "bin": {"sdk": "bin/sdk.js"}, "engines": {"node": ">=18"}}, + "node_modules/@novox/ui": {"version": "1.1.4", "dev": true, "funding": [{"type": "x", "url": "y"}]}, + "node_modules/left-pad": {"version": "1.3.0"} + }, + "dependencies": { + "@novox/mesh-sdk": {"version": "0.1.3", "requires": {"left-pad": "^1.3.0"}}, + "@novox/ui": {"version": "1.1.4", "dev": true}, + "left-pad": {"version": "1.3.0"} + } +}` + +const lockV3 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 3, "requires": true, + "packages": { + "": {"name": "app", "workspaces": ["packages/*"], "dependencies": {"@novox/mesh-sdk": "^0.1.0", "sdk": "npm:@novox/ui@^2.0.0"}}, + "node_modules/@novox/mesh-sdk": {"version": "0.1.5", "license": "MIT", "dependencies": {"@novox/ui": "^1.0.0"}}, + "node_modules/@novox/ui": {"version": "1.0.9", "devDependencies": {"typescript": "^5"}}, + "node_modules/a/node_modules/@novox/mesh-sdk": {"version": "0.1.1", "dependencies": {"left-pad": "1.x"}}, + "node_modules/a/node_modules/b/node_modules/@novox/ui": {"version": "1.0.2"}, + "node_modules/sdk": {"name": "@novox/ui", "version": "2.0.4", "resolved": "https://forge.example/u.tgz"}, + "node_modules/@novox/workspace-thing": {"resolved": "packages/thing", "link": true}, + "node_modules/@novox/linked": {"resolved": "packages/linked", "link": true, "version": "9.9.9"}, + "packages/thing": {"name": "@novox/workspace-thing", "version": "0.0.1", "dependencies": {"@novox/ui": "^1.0.0"}}, + "node_modules/left-pad": {"version": "1.3.0", "peerDependencies": {"x": "*"}, "optionalDependencies": {"y": "1"}} + } +}` + +var knownLocks = map[string]bool{"@novox/mesh-sdk": true, "@novox/ui": true, "@novox/workspace-thing": true, "@novox/linked": true} + +func TestALockfileVersion1NamesEveryNestedVersion(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV1), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + if pins(got) != "@novox/mesh-sdk@0.1.0 @novox/mesh-sdk@0.1.2 @novox/ui@1.0.0 @novox/ui@2.0.1" { + t.Fatalf("v1: %s", pins(got)) + } +} + +func TestALockfileVersion2ReadsBothHalves(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV2), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + // Each half names both; a version is kept once it is named, so naming it twice is harmless. + if pins(got) != "@novox/mesh-sdk@0.1.3 @novox/mesh-sdk@0.1.3 @novox/ui@1.1.4 @novox/ui@1.1.4" { + t.Fatalf("v2: %s", pins(got)) + } +} + +func TestALockfileVersion3ReadsPathsAliasesAndSkipsLinks(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV3), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + want := "@novox/mesh-sdk@0.1.1 @novox/mesh-sdk@0.1.5 @novox/ui@1.0.2 @novox/ui@1.0.9 @novox/ui@2.0.4" + if pins(got) != want { + t.Fatalf("v3: %s\nwant %s", pins(got), want) + } +} + +func TestAnEntryOfAnUnknownShapeIsSaidAndTheRestIsRead(t *testing.T) { + lock := `{"lockfileVersion":3,"packages":{ + "node_modules/@novox/ui":{"version":"1.0.0"}, + "node_modules/@novox/mesh-sdk":{"version":{"odd":true}}}, + "dependencies":{"@novox/ui":"not an object"}}` + got, err := ReadLock("package-lock.json", []byte(lock), "r", knownLocks) + if pins(got) != "@novox/ui@1.0.0" { + t.Fatalf("read %s", pins(got)) + } + if err == nil || !strings.Contains(err.Error(), "2 entries not read") || + !strings.Contains(err.Error(), `packages["node_modules/@novox/mesh-sdk"]`) { + t.Fatalf("got %v", err) + } +} + +func mustYarn(t *testing.T, content string) []Pinned { + t.Helper() + out, err := readYarnLock([]byte(content), "r", knownHere) + if err != nil { + t.Fatal(err) + } + return out +} + +// A yarn lockfile with a line longer than the scanner holds is an error, not a file read to its end. +func TestAYarnLockfileCutShortIsAnError(t *testing.T) { + long := "\"@novox/mesh-sdk@^0.1.0\":\n version \"0.1.4\"\n# " + strings.Repeat("x", 2<<20) + "\n\"@novox/ui@^2\":\n version \"2.0.7\"\n" + if _, err := readYarnLock([]byte(long), "r", knownHere); err == nil { + t.Fatal("a lockfile the scanner stopped in was read as whole") + } +} diff --git a/modules/gitea/cmd/npm-registry/retention.go b/modules/gitea/cmd/npm-registry/retention.go index 886e2706..4f9915db 100644 --- a/modules/gitea/cmd/npm-registry/retention.go +++ b/modules/gitea/cmd/npm-registry/retention.go @@ -187,11 +187,10 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S continue } got, err := ReadLock(base, content, where, known) + ps = append(ps, got...) if err != nil { notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err)) - continue } - ps = append(ps, got...) } mu.Lock() defer mu.Unlock() @@ -356,6 +355,7 @@ type Candidate struct { Version string `json:"version"` Published time.Time `json:"published"` Bytes int64 `json:"bytes"` + SizeError string `json:"size_error,omitempty"` } // Plan is what retention would delete, and — for a real run — what it did. @@ -365,8 +365,10 @@ type Plan struct { Why string `json:"why,omitempty"` Candidates []Candidate `json:"would_delete"` Bytes int64 `json:"bytes"` - Deleted []string `json:"deleted,omitempty"` - Refused []string `json:"refused,omitempty"` + // RegistryBytes is every version's size together, deleted or kept: what retention is measured against. + RegistryBytes int64 `json:"registry_bytes"` + Deleted []string `json:"deleted,omitempty"` + Refused []string `json:"refused,omitempty"` } // retention works out the plan, and carries it out when dryRun is false. @@ -374,21 +376,28 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool if !dryRun && strings.TrimSpace(why) == "" { return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done") } - s, err := survey(ctx, f, only, keep, func(i Item) bool { return !i.Kept }) + // Every version is measured, kept or not, so no size in the answer is a 0 nobody asked for. + s, err := survey(ctx, f, only, keep, func(Item) bool { return true }) if err != nil { return nil, err } p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}} for _, pkg := range s.Packages { for _, v := range pkg.Versions { + p.RegistryBytes += v.Bytes if !v.Kept { - p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes}) + p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes, v.SizeError}) p.Bytes += v.Bytes } } } - s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages; %d would be deleted, %s", - s.Repositories, countVersions(s), len(s.Packages), len(p.Candidates), mib(p.Bytes))) + s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages, %s; %d would be deleted, %s", + s.Repositories, countVersions(s), len(s.Packages), mib(p.RegistryBytes), len(p.Candidates), mib(p.Bytes))) + s.Said = append(s.Said, sizesSaid(s)...) + if len(s.NotRead) > 0 { + s.Said = append(s.Said, fmt.Sprintf("%d files were not read whole (files_not_read): what they name is not "+ + "known, so a real run deletes nothing", len(s.NotRead))) + } if len(s.Unread) > 0 { s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+ "known, so a real run deletes nothing", len(s.Unread))) @@ -401,6 +410,12 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool return p, fmt.Errorf("refused: %d repositories or files could not be read, so what depends on what is not "+ "known; nothing was deleted (first: %s)", len(s.Unread), s.Unread[0]) } + // A file that was fetched but not read whole is a file of its repository that could not be read + // (novox/hq ADR 0251 §4): a version only it names would be deleted while in use. + if len(s.NotRead) > 0 { + return p, fmt.Errorf("refused: %d files were not read whole, so what they name is not known; nothing was "+ + "deleted (first: %s)", len(s.NotRead), s.NotRead[0]) + } del, cancel := context.WithTimeout(context.Background(), budget) defer cancel() for _, c := range p.Candidates { @@ -422,4 +437,22 @@ func countVersions(s *Survey) int { return n } +// sizesSaid says, when any version's size could not be read, how many and the first reason: those versions +// count 0 bytes, so every total is at least, not exactly. +func sizesSaid(s *Survey) []string { + var failed []string + for _, p := range s.Packages { + for _, v := range p.Versions { + if v.SizeError != "" { + failed = append(failed, fmt.Sprintf("%s@%s: %s", p.Name, v.Version, v.SizeError)) + } + } + } + if len(failed) == 0 { + return nil + } + return []string{fmt.Sprintf("the size of %d versions could not be read, so the bytes said are at least, not exactly "+ + "(first: %s)", len(failed), failed[0])} +} + func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) } diff --git a/modules/gitea/cmd/npm-registry/retention_test.go b/modules/gitea/cmd/npm-registry/retention_test.go index d81e3c4a..9c252803 100644 --- a/modules/gitea/cmd/npm-registry/retention_test.go +++ b/modules/gitea/cmd/npm-registry/retention_test.go @@ -24,6 +24,8 @@ type fakeForge struct { tags map[string]map[string]string // package -> dist-tags repos map[string]map[string]string // repository -> path -> content broken map[string]bool // repositories whose tree answers 500 + files map[string][]int64 // package@version -> its files' sizes (default 1 MiB + 1 KiB) + noSize map[string]bool // package@version whose files answer 500 mu sync.Mutex deleted []string password string @@ -74,7 +76,19 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { name, _ := url.PathUnescape(m[1]) version, _ := url.PathUnescape(m[2]) if m[3] != "" { - writeJSON(w, []map[string]any{{"name": "a.tgz", "size": 1 << 20}, {"name": "b", "size": 1024}}) + if f.noSize[name+"@"+version] { + http.Error(w, "storage unavailable", http.StatusInternalServerError) + return + } + sizes, has := f.files[name+"@"+version] + if !has { + sizes = []int64{1 << 20, 1024} + } + var files []map[string]any + for i, n := range sizes { + files = append(files, map[string]any{"id": i + 1, "size": n, "name": fmt.Sprintf("f%d", i), "md5": "x"}) + } + writeJSON(w, files) return } if r.Method != http.MethodDelete { @@ -391,3 +405,108 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Error("ran without a forge") } } + +func TestEveryVersionIsMeasuredAndASizeNotReadIsSaid(t *testing.T) { + f := aRegistry(t) + f.files = map[string][]int64{"@acme/sdk@0.1.1": {1000, 234}, "@acme/sdk@0.3.0": {5000}} + f.noSize = map[string]bool{"@acme/sdk@0.2.0": true} + p, err := retention(context.Background(), f.serve(), "@acme/sdk", 3, true, "") + if err != nil { + t.Fatal(err) + } + d := decided(p.Survey) + // A kept version is measured too: no 0 that nobody asked for. + if got := d["@acme/sdk@0.3.0"]; !got.Kept || got.Bytes != 5000 || got.SizeError != "" { + t.Errorf("kept 0.3.0: %+v", got) + } + if got := d["@acme/sdk@0.1.1"]; got.Bytes != 1234 { + t.Errorf("0.1.1: %+v", got) + } + broken := d["@acme/sdk@0.2.0"] + if broken.Bytes != 0 || !strings.Contains(broken.SizeError, "500") { + t.Errorf("0.2.0: %+v", broken) + } + var candidate *Candidate + for i := range p.Candidates { + if p.Candidates[i].Version == "0.2.0" { + candidate = &p.Candidates[i] + } + } + if candidate == nil || candidate.SizeError == "" { + t.Errorf("the candidate does not carry its size error: %+v", p.Candidates) + } + // Would delete 0.1.1 (1234), 0.1.2 and 0.2.1 (1 MiB + 1 KiB each) and 0.2.0 (unknown). + if want := int64(1234 + 2*(1<<20+1024)); p.Bytes != want { + t.Errorf("bytes %d, want %d", p.Bytes, want) + } + if p.RegistryBytes <= p.Bytes { + t.Errorf("registry bytes %d not above %d", p.RegistryBytes, p.Bytes) + } + said := strings.Join(p.Said, "\n") + if !strings.Contains(said, "the size of 1 versions could not be read") || !strings.Contains(said, "@acme/sdk@0.2.0") { + t.Errorf("said %q", said) + } +} + +func TestALockfileOfEveryShapeKeepsWhatItPins(t *testing.T) { + f := aRegistry(t) + f.repos["acme/v2"] = map[string]string{"package-lock.json": `{"lockfileVersion":2,"packages":{ + "":{"name":"v2","dependencies":{"@acme/sdk":"^0.2.0"}}, + "node_modules/@acme/sdk":{"version":"0.2.0","dependencies":{"left-pad":"^1.0.0"}}}, + "dependencies":{"@acme/sdk":{"version":"0.2.0","requires":{"left-pad":"^1.0.0"}}}}`} + s, err := survey(context.Background(), f.serve(), "@acme/sdk", 1, nil) + if err != nil { + t.Fatal(err) + } + if len(s.NotRead) != 0 { + t.Fatalf("not read %v", s.NotRead) + } + if got := decided(s)["@acme/sdk@0.2.0"]; !contains(got.Why, "a lockfile names it: acme/v2:package-lock.json") { + t.Fatalf("0.2.0: %+v", got) + } +} + +func TestAnUnreadableLockfileStopsARealRunBeforeAnythingIsDeleted(t *testing.T) { + f := aRegistry(t) + f.repos["acme/odd"] = map[string]string{"package-lock.json": `{"lockfileVersion":3,"packages":{`} + p, err := retention(context.Background(), f.serve(), "", 3, false, "tidy") + if err == nil || !strings.Contains(err.Error(), "nothing was deleted") || !strings.Contains(err.Error(), "acme/odd:package-lock.json") { + t.Fatalf("got %v", err) + } + if len(f.deleted) != 0 { + t.Fatalf("deleted %v while a lockfile was not read", f.deleted) + } + if p == nil || len(p.NotRead) != 1 || len(p.Deleted) != 0 { + t.Fatalf("plan %+v", p) + } + // And the dry run says so, without failing, and still says what would go. + dry, err := retention(context.Background(), f.serve(), "", 3, true, "") + if err != nil || len(dry.NotRead) != 1 || len(dry.Candidates) == 0 { + t.Fatalf("dry run: %v %+v", err, dry) + } + if !strings.Contains(strings.Join(dry.Said, "\n"), "1 files were not read whole") { + t.Errorf("said %v", dry.Said) + } +} + +// An answer larger than the cap is refused, never parsed as if the cut body were whole; one exactly at +// the cap is read. +func TestAnAnswerTheCapCutIsRefused(t *testing.T) { + size := maxBody + 1 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(make([]byte, size)) + })) + t.Cleanup(srv.Close) + file := filepath.Join(t.TempDir(), "admin.secret") + if err := os.WriteFile(file, []byte("x\n"), 0o600); err != nil { + t.Fatal(err) + } + f := &Forge{URL: srv.URL, User: "admin", PasswordFile: file, Owner: "acme"} + if _, err := f.do(context.Background(), http.MethodGet, "/big"); err == nil || !strings.Contains(err.Error(), "not read whole") { + t.Fatalf("a cut answer: %v", err) + } + size = maxBody + if body, err := f.do(context.Background(), http.MethodGet, "/big"); err != nil || len(body) != maxBody { + t.Fatalf("an answer at the cap: %d bytes, %v", len(body), err) + } +}