diff --git a/modules/gitea/README.md b/modules/gitea/README.md index ab42d456..8d474811 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -45,28 +45,38 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag | `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path | **Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266): -never the repository's default branch, never a branch a protection rule names (a glob rule covers by the same -matcher as `guard.ts`). The rules are read before anything is sent; rules that cannot be read refuse the +never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the +forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob +(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob +the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the commit. Nothing about merging changes: a merge still needs an independent review at the head and both required checks green, through `gitea_merge_pull_request`. **How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the ref holds is updated, one it does not is created, and an update or delete carries the blob read just -before, so a file someone changed meanwhile is refused by the forge rather than overwritten. A `patch` goes -to the forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than -the file. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the +before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at +the commit the branch (or, for a new branch, its base) was read at, never at its name. A `patch` goes to the +forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the +file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the +one the forge says it made the commit on; when the branch moved between the read and the commit, the answer +says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a branch not at the commit the forge named is a failure, never a success. **Code search** has no endpoint on the forge, so the ref's archive (`/archive/{ref}.tar.gz`, at most 64 MiB) is read once and searched in memory; a binary file is passed over, a file over 4 MiB is named under -`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. +`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A +search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. -It reaches the forge as the npm-registry bundle does (below), and one tool call takes at most 50 seconds. +It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one +tool call takes at most 50 seconds. `go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on a new branch from its base; an existing branch committed on; the default branch, a protected branch and unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another -commit failing; every malformed ask refused; and each read summarised. +commit failing; a branch or base that moved under the commit said, with the parent the forge used; a file +changed under the commit refused; a commit over its size limits refused; protection globs matched as the +forge matches them; a code search stopped at its deadline; every malformed ask refused; and each read +summarised. ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) diff --git a/modules/gitea/cmd/forge/commit.go b/modules/gitea/cmd/forge/commit.go index f60c33b6..6976da57 100644 --- a/modules/gitea/cmd/forge/commit.go +++ b/modules/gitea/cmd/forge/commit.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strings" "unicode/utf8" ) @@ -52,8 +53,14 @@ type CommittedFile struct { Operation string `json:"operation"` } -// maxFiles is the most files one commit carries. -const maxFiles = 200 +// What one commit carries at most (novox/hq issue 539): files, one file's content, one patch, and every +// file's content together, in bytes as committed (content_base64 decoded). Larger work is several commits. +const ( + maxFiles = 200 + maxFileBytes = 4 << 20 + maxPatchBytes = 4 << 20 + maxCommitBytes = 16 << 20 +) // readCommitAsk reads the tool's arguments. func readCommitAsk(args map[string]any) (CommitAsk, error) { @@ -109,8 +116,11 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) { return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent") case len(a.Files) > maxFiles: return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles) + case len(a.Patch) > maxPatchBytes: + return a, refuse("the patch is %s, more than the %s one commit carries: split it into several commits. Nothing was sent", mib(len(a.Patch)), mib(maxPatchBytes)) } seen := map[string]bool{} + total := 0 for i, fc := range a.Files { if strings.HasPrefix(fc.Path, "/") { return a, refuse("files[%d].path %q is relative to the repository's root, without a leading slash. Nothing was sent", i, fc.Path) @@ -134,15 +144,35 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) { if fc.Content != nil && !utf8.ValidString(*fc.Content) { return a, refuse("files[%d].content (%s) is not UTF-8 text: give it as content_base64. Nothing was sent", i, fc.Path) } - if fc.ContentBase64 != nil { - if _, err := base64.StdEncoding.DecodeString(*fc.ContentBase64); err != nil { + size := 0 + switch { + case fc.Content != nil: + size = len(*fc.Content) + case fc.ContentBase64 != nil: + if base64.StdEncoding.DecodedLen(len(*fc.ContentBase64)) > maxFileBytes+3 { + size = base64.StdEncoding.DecodedLen(len(*fc.ContentBase64)) + break + } + raw, err := base64.StdEncoding.DecodeString(*fc.ContentBase64) + if err != nil { return a, refuse("files[%d].content_base64 (%s) is not base64: %v. Nothing was sent", i, fc.Path, err) } + size = len(raw) } + if size > maxFileBytes { + return a, refuse("files[%d] (%s) is %s, more than the %s one file of a commit may be. Nothing was sent", i, fc.Path, mib(size), mib(maxFileBytes)) + } + total += size + } + if total > maxCommitBytes { + return a, refuse("the files are %s together, more than the %s one commit carries: split them into several commits. Nothing was sent", mib(total), mib(maxCommitBytes)) } return a, nil } +// mib says a size in bytes in MiB, to a tenth. +func mib(n int) string { return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) } + // targetRefused says why a commit may not go on a branch: the default branch, or one a protection rule names. func targetRefused(target, defaultBranch string, rules []Protection) error { if target == defaultBranch { @@ -172,7 +202,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { return nil, err } var r Repo - if err := f.getJSON(ctx, repo, &r); err != nil { + if err := f.GetJSON(ctx, repo, &r); err != nil { return nil, err } if r.DefaultBranch == "" { @@ -180,7 +210,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { } // Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed). var rules []Protection - if err := f.getJSON(ctx, repo+"/branch_protections", &rules); err != nil { + if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil { return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) } if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil { @@ -192,9 +222,13 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { if err != nil { return nil, err } - ref := a.Branch // where the files are read and the commit goes + // The files are judged at the commit just read, never at a branch name that could move between two + // reads (novox/hq issue 539): a file the branch changes meanwhile is refused by the forge, whose update + // and delete carry the blob read here. + var ref string if existing != nil { out.Parent = existing.Commit.ID + ref = existing.Commit.ID if a.Base != "" { out.Said = append(out.Said, fmt.Sprintf("%s exists, at %s: base %s was not used", a.Branch, short(existing.Commit.ID), a.Base)) } @@ -204,7 +238,6 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { if out.Base == "" { out.Base = r.DefaultBranch } - ref = out.Base b, err := f.branch(ctx, repo, out.Base) if err != nil { return nil, err @@ -213,8 +246,6 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { return nil, refuse("%s does not exist, and its base %s is not a branch of %s/%s. Nothing was sent", a.Branch, out.Base, a.Owner, a.Repo) } out.Parent = b.Commit.ID - // The files are judged at the commit just read; a file the base changes meanwhile is refused by the - // forge, whose update and delete carry the blob read here. ref = b.Commit.ID } @@ -227,14 +258,14 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { on["branch"] = a.Branch } + // The forge takes no commit to build on, only a branch (Gitea 1.27: neither /contents nor /diffpatch + // carries a last commit), so the parent is read from what it made, never assumed (novox/hq issue 539). var made struct { - Commit struct { - SHA string `json:"sha"` - } `json:"commit"` + Commit madeCommit `json:"commit"` } if a.Patch != "" { on["content"] = a.Patch - if err := f.sendJSON(ctx, http.MethodPost, repo+"/diffpatch", on, &made); err != nil { + if err := f.SendJSON(ctx, http.MethodPost, repo+"/diffpatch", on, &made); err != nil { return nil, fmt.Errorf("the forge did not take the patch: %w", err) } } else { @@ -266,7 +297,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { out.Files = append(out.Files, CommittedFile{Path: fc.Path, Operation: op["operation"].(string)}) } on["files"] = ops - if err := f.sendJSON(ctx, http.MethodPost, repo+"/contents", on, &made); err != nil { + if err := f.SendJSON(ctx, http.MethodPost, repo+"/contents", on, &made); err != nil { return nil, fmt.Errorf("the forge did not take the commit: %w", err) } } @@ -274,6 +305,25 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { return nil, fmt.Errorf("the forge answered the commit on %s without saying the commit it made: read the branch (gitea_list_branches) before trying again", a.Branch) } out.Commit = made.Commit.SHA + read := out.Parent + parent, err := f.parentOf(ctx, repo, made.Commit) + if err != nil { + return nil, fmt.Errorf("the forge made %s on %s, but which commit it went on could not be read: %w", out.Commit, a.Branch, err) + } + out.Parent = parent + if parent != read { + moved := "branch " + a.Branch + if out.Created { + moved = "base " + out.Base + } + how := "each file it changes was checked against " + short(read) + " by its blob, so none of them changed in between" + if a.Patch != "" { + how = "the forge applied the patch there, and it refuses a patch whose context changed" + } + out.Said = append(out.Said, fmt.Sprintf("the %s moved from %s to %s between the read and the commit, so the "+ + "commit went on %s: %s. What else changed in between: gitea_compare %s...%s", + moved, short(read), short(parent), short(parent), how, short(read), short(parent))) + } // Evidence over assertion: the branch is read back, and it must be at the commit the forge said. after, err := f.branch(ctx, repo, a.Branch) @@ -294,6 +344,30 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { return out, nil } +// madeCommit is the commit the forge answers a write with. +type madeCommit struct { + SHA string `json:"sha"` + Parents []struct { + SHA string `json:"sha"` + } `json:"parents"` +} + +// parentOf is the commit the forge made its commit on: from its answer, or read from the commit when the +// answer does not say it. +func (f *Forge) parentOf(ctx context.Context, repo string, made madeCommit) (string, error) { + if len(made.Parents) == 0 { + var read madeCommit + if err := f.GetJSON(ctx, repo+"/git/commits/"+url.PathEscape(made.SHA)+"?stat=false&verification=false&files=false", &read); err != nil { + return "", err + } + made = read + } + if len(made.Parents) != 1 || made.Parents[0].SHA == "" { + return "", fmt.Errorf("the forge says %s has %d parents, not one", made.SHA, len(made.Parents)) + } + return made.Parents[0].SHA, nil +} + // blobAt is the blob sha of a file at a ref, "" when there is no file there. func (f *Forge) blobAt(ctx context.Context, repo, path, ref string) (string, error) { seg, err := segments("file path", path) @@ -305,7 +379,7 @@ func (f *Forge) blobAt(ctx context.Context, repo, path, ref string) (string, err SHA string `json:"sha"` } var list []any - body, err := f.do(ctx, http.MethodGet, repo+"/contents/"+seg+"?ref="+queryEscape(ref), nil, maxBody) + body, err := f.Do(ctx, http.MethodGet, repo+"/contents/"+seg+"?ref="+queryEscape(ref), nil, maxBody) if err != nil { if errors.Is(err, errNotFound) { return "", nil diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index 940e08a0..3637116d 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -1,156 +1,43 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" "fmt" - "io" - "net/http" "net/url" - "os" "regexp" "strings" - "time" + + "gitea/internal/forgehttp" ) -// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account, -// which the vault delivered into a file this module owns (novox/hq ADR 0086), by basic authentication — the -// same way the npm-registry bundle beside it does. The password is read when a call is made and never logged, -// answered or put in an error. +// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account +// (internal/forgehttp, shared with the npm-registry bundle beside it). // Forge is the forge's HTTP interface. type Forge struct { - URL string // e.g. http://127.0.0.1:3000 - User string - PasswordFile string - HTTP *http.Client + forgehttp.Client } // ForgeFromEnv is the forge as the manifest hands it to this bundle. func ForgeFromEnv() (*Forge, error) { - f := &Forge{ - URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), - User: os.Getenv("MESH_GITEA_ADMIN_USER"), - PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), - } - var missing []string - for _, kv := range [][2]string{{"MESH_GITEA_URL", f.URL}, {"MESH_GITEA_ADMIN_USER", f.User}, - {"MESH_GITEA_ADMIN_PASSWORD_FILE", f.PasswordFile}} { - if kv[1] == "" { - missing = append(missing, kv[0]) - } - } + c, missing := forgehttp.FromEnv() if len(missing) > 0 { return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest", strings.Join(missing, ", ")) } - return f, nil + c.Timeout = deadline + return &Forge{Client: c}, nil } // errNotFound is the forge answering 404. -var errNotFound = errors.New("not found") - -// statusError is an answer the forge gave that is not a success, said without the request's credential. -type statusError struct { - method, path string - status int - said string -} - -func (e *statusError) Error() string { - return fmt.Sprintf("the forge answered %d to %s %s: %s", e.status, e.method, e.path, e.said) -} - -func (f *Forge) client() *http.Client { - if f.HTTP != nil { - return f.HTTP - } - return &http.Client{Timeout: 50 * time.Second} -} - -func (f *Forge) password() (string, error) { - raw, err := os.ReadFile(f.PasswordFile) - if err != nil { - // The path is the manifest's, not a secret; the content never appears. - return "", fmt.Errorf("the forge's admin password cannot be read from its file: %w", err) - } - return strings.TrimSpace(string(raw)), nil -} +var errNotFound = forgehttp.ErrNotFound // maxBody is the largest JSON answer read from the forge; maxArchive the largest archive code search reads. const ( - maxBody = 32 << 20 + maxBody = forgehttp.MaxBody maxArchive = 64 << 20 ) -// do sends one request and answers the body, read up to limit; 404 is errNotFound. A body the limit cut is -// refused, never read as if it were whole. -func (f *Forge) do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { - password, err := f.password() - if err != nil { - return nil, err - } - var body io.Reader - if payload != nil { - raw, err := json.Marshal(payload) - if err != nil { - return nil, err - } - body = bytes.NewReader(raw) - } - req, err := http.NewRequestWithContext(ctx, method, f.URL+path, body) - if err != nil { - return nil, err - } - req.SetBasicAuth(f.User, password) - req.Header.Set("Accept", "application/json") - if payload != nil { - req.Header.Set("Content-Type", "application/json") - } - res, err := f.client().Do(req) - if err != nil { - return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err) - } - defer res.Body.Close() - got, err := io.ReadAll(io.LimitReader(res.Body, limit+1)) - if err != nil { - return nil, err - } - if int64(len(got)) > limit { - return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) - } - switch { - case res.StatusCode == http.StatusNotFound: - return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound) - case res.StatusCode >= 300: - said := strings.TrimSpace(string(got)) - if len(said) > 300 { - said = said[:300] + "…" - } - return nil, &statusError{method, path, res.StatusCode, said} - } - return got, nil -} - -func (f *Forge) getJSON(ctx context.Context, path string, into any) error { - return f.sendJSON(ctx, http.MethodGet, path, nil, into) -} - -func (f *Forge) sendJSON(ctx context.Context, method, path string, payload, into any) error { - body, err := f.do(ctx, method, path, payload, maxBody) - if err != nil { - return err - } - if into == nil { - return nil - } - if err := json.Unmarshal(body, into); err != nil { - return fmt.Errorf("the forge's answer to %s %s is not readable: %w", method, path, err) - } - return nil -} - // Refused is a refusal of policy or of an argument, in the words the caller sees. type Refused struct{ msg string } @@ -204,36 +91,6 @@ func (p Protection) name() string { return p.BranchName } -// ruleCovers is whether a protection rule's name — a branch, or a glob of them (`*`, `**`, `?`) — covers a -// branch: the TypeScript bundle's ruleCovers (guard.ts), which gitea_api's file writes are judged by. -func ruleCovers(rule, branch string) bool { - if rule == branch { - return true - } - if !strings.ContainsAny(rule, "*?[") { - return false - } - var re strings.Builder - re.WriteString("^") - for i := 0; i < len(rule); i++ { - c := rule[i] - switch { - case c == '*' && i+1 < len(rule) && rule[i+1] == '*': - re.WriteString(".*") - i++ - case c == '*': - re.WriteString("[^/]*") - case c == '?': - re.WriteString("[^/]") - default: - re.WriteString(regexp.QuoteMeta(string(c))) - } - } - re.WriteString("$") - ok, err := regexp.MatchString(re.String(), branch) - return err == nil && ok -} - // Branch is one branch with the commit it points at. type Branch struct { Name string `json:"name"` @@ -249,7 +106,7 @@ func (f *Forge) branch(ctx context.Context, repo, name string) (*Branch, error) return nil, err } var b Branch - if err := f.getJSON(ctx, repo+"/branches/"+seg, &b); err != nil { + if err := f.GetJSON(ctx, repo+"/branches/"+seg, &b); err != nil { if errors.Is(err, errNotFound) { return nil, nil } diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go index 5e5cd039..bee66856 100644 --- a/modules/gitea/cmd/forge/forge_test.go +++ b/modules/gitea/cmd/forge/forge_test.go @@ -13,9 +13,12 @@ import ( "net/http/httptest" "os" "path/filepath" + "regexp" "strings" "sync" "testing" + + "gitea/internal/forgehttp" ) // fakeForge is the forge's HTTP interface as far as these tools use it. @@ -37,6 +40,16 @@ type fakeForge struct { comments map[string][]map[string]any commits []map[string]any dir []map[string]any + // moves: a branch moved by someone else right after it is first read — to a commit, with its tree. + moves map[string]move + noParents bool // the forge's answer to a write does not say the parents + parentsBroken bool // and GET git/commits fails + parents map[string]string // commit -> parent, as GET git/commits says it +} + +type move struct { + to string + tree map[string]string } func newFake(t *testing.T) *fakeForge { @@ -62,7 +75,7 @@ func (f *fakeForge) serve() *Forge { } srv := httptest.NewServer(http.HandlerFunc(f.handle)) f.t.Cleanup(srv.Close) - return &Forge{URL: srv.URL, User: "mesh-admin", PasswordFile: pw} + return &Forge{Client: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}} } func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { @@ -92,6 +105,11 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { return } reply(map[string]any{"name": name, "commit": map[string]any{"id": c}}) + if m, ok := f.moves[name]; ok { + delete(f.moves, name) + f.trees[m.to] = m.tree + f.branches[name] = m.to + } case r.Method == "GET" && (path == repo+"/contents" || path == repo+"/contents/docs"): reply(f.dir) case r.Method == "GET" && strings.HasPrefix(path, repo+"/contents/"): @@ -148,19 +166,28 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { if !f.staleRead { f.branches[target] = made } - reply(map[string]any{"commit": map[string]any{"sha": made}}) + reply(map[string]any{"commit": f.made(made, from)}) case r.Method == "POST" && path == repo+"/diffpatch": var body map[string]any _ = json.NewDecoder(r.Body).Decode(&body) f.patches = append(f.patches, body) f.next++ made := fmt.Sprintf("p%d", f.next) + from := f.branches[body["branch"].(string)] target := body["branch"].(string) if nb, ok := body["new_branch"].(string); ok && nb != "" { target = nb } f.branches[target] = made - reply(map[string]any{"commit": map[string]any{"sha": made}}) + reply(map[string]any{"commit": f.made(made, from)}) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/git/commits/"): + sha := strings.TrimPrefix(path, repo+"/git/commits/") + p, ok := f.parents[sha] + if !ok || f.parentsBroken { + http.NotFound(w, r) + return + } + reply(map[string]any{"sha": sha, "parents": []any{map[string]any{"sha": p}}}) case r.Method == "GET" && path == repo+"/commits": f.writes = append(f.writes, map[string]any{"query": r.URL.RawQuery}) reply(f.commits) @@ -180,6 +207,18 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { } } +// made is the forge's answer about a commit it made on a parent, with the parents unless noParents. +func (f *fakeForge) made(sha, parent string) map[string]any { + if f.parents == nil { + f.parents = map[string]string{} + } + f.parents[sha] = parent + if f.noParents { + return map[string]any{"sha": sha} + } + return map[string]any{"sha": sha, "parents": []any{map[string]any{"sha": parent}}} +} + func filesArg(files ...map[string]any) []any { out := make([]any, len(files)) for i, f := range files { @@ -367,6 +406,13 @@ func TestRuleCoversIsTheGuardsRule(t *testing.T) { {"main", "main", true}, {"main", "mainx", false}, {"release/*", "release/1.0", true}, {"release/*", "release/a/b", false}, {"release/**", "release/a/b", true}, {"v?", "v1", true}, {"a.b", "axb", false}, + // As the forge decides it (novox/hq issue 539): classes, alternatives, escapes, a plain name in any case. + {"release-[ab]", "release-a", true}, {"release-[ab]", "release-c", false}, {"release-[!ab]", "release-c", true}, + {"v[0-9].x", "v7.x", true}, {"v[0-9].x", "vA.x", false}, {"{main,master}", "master", true}, + {"{main,master}", "mainline", false}, {"rel/{a,b}/*", "rel/b/1", true}, {"rel/{a,b}/*", "rel/c/1", false}, + {"Main", "main", true}, {"main", "MAIN", true}, {"Feat/*", "feat/x", false}, {`a\*`, "a*", true}, + {`a\*`, "ab", false}, {"a+b", "a+b", true}, {"a+b", "aab", false}, {"{open", "{open", true}, + {"[open", "[open", true}, {"[open", "o", false}, } { if got := ruleCovers(c.rule, c.branch); got != c.want { t.Errorf("%s covers %s: %v", c.rule, c.branch, got) @@ -581,3 +627,115 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { t.Error("the manifest builds no cmd/forge bundle") } } + +// novox/hq issue 539: the forge takes a branch, never a commit to build on, so the answer's parent is the one +// the forge made the commit on, and a branch that moves between the read and the commit is said. +func TestAnExistingBranchMovedUnderTheCommitIsSaidAndItsParentIsTheOneTheForgeUsed(t *testing.T) { + f := newFake(t) + f.moves = map[string]move{"fix/1-old": {to: "c2", tree: map[string]string{"README.md": "hello\n", "docs/a.md": "a on fix\n", "other.md": "x\n"}}} + got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/1-old", + "files": filesArg(map[string]any{"path": "docs/a.md", "content": "a fixed\n"})})) + if err != nil { + t.Fatal(err) + } + if got.Parent != "c2" || f.branches["fix/1-old"] != got.Commit { + t.Fatalf("parent %s, want c2 (the commit the forge built on): %+v", got.Parent, got) + } + said := strings.Join(got.Said, " ") + if !strings.Contains(said, "moved from c1 to c2") || !strings.Contains(said, "after c2") { + t.Fatalf("the move was not said: %v", got.Said) + } +} + +func TestAnExistingBranchsFilesAreJudgedAtTheCommitReadNotAtWhereTheBranchMovedTo(t *testing.T) { + f := newFake(t) + // Someone else changes docs/a.md on the branch between the read and the commit: that change is not + // overwritten by a blob read at the branch's new head. + f.moves = map[string]move{"fix/1-old": {to: "c2", tree: map[string]string{"README.md": "hello\n", "docs/a.md": "theirs\n"}}} + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/1-old", + "files": filesArg(map[string]any{"path": "docs/a.md", "content": "mine\n"})})) + if err == nil || !strings.Contains(err.Error(), "sha does not match") { + t.Fatalf("a file changed under the commit was overwritten: %v", err) + } + if f.trees["c2"]["docs/a.md"] != "theirs\n" || f.branches["fix/1-old"] != "c2" { + t.Fatalf("the branch was changed: %v", f.branches) + } +} + +func TestAPatchOnABaseThatMovedNamesTheCommitTheForgeAppliedItTo(t *testing.T) { + f := newFake(t) + f.moves = map[string]move{"main": {to: "c9", tree: map[string]string{"README.md": "hello\n"}}} + patch := "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n" + got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/6", "patch": patch})) + if err != nil { + t.Fatal(err) + } + if got.Parent != "c9" || !strings.Contains(strings.Join(got.Said, " "), "base main moved from c0 to c9") { + t.Fatalf("%+v", got) + } +} + +func TestAParentTheAnswerDoesNotSayIsReadFromTheCommit(t *testing.T) { + f := newFake(t) + f.noParents = true + got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/7", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err != nil || got.Parent != "c0" { + t.Fatalf("%v %+v", err, got) + } + // A parent that cannot be read is a failure, said with the commit that was made. + f = newFake(t) + f.noParents, f.parentsBroken = true, true + _, err = commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/8", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "the forge made n1 on fix/8, but which commit it went on could not be read") { + t.Fatalf("%v", err) + } +} + +func TestACommitLargerThanItsLimitsIsRefusedBeforeAnythingIsSent(t *testing.T) { + big := strings.Repeat("x", maxFileBytes+1) + quarter := strings.Repeat("y", maxFileBytes) + many := filesArg() + for i := 0; i < maxCommitBytes/maxFileBytes+1; i++ { + many = append(many, map[string]any{"path": fmt.Sprintf("f%d", i), "content": quarter}) + } + for name, args := range map[string]map[string]any{ + "one file": {"files": filesArg(map[string]any{"path": "x", "content": big})}, + "one file b64": {"files": filesArg(map[string]any{"path": "x", "content_base64": base64.StdEncoding.EncodeToString([]byte(big))})}, + "the patch": {"patch": "diff --git a/x b/x\n" + big}, + "all files": {"files": many}, + } { + full := map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b"} + for k, v := range args { + full[k] = v + } + _, err := readCommitAsk(full) + if err == nil || !strings.Contains(err.Error(), "MiB") || !strings.Contains(err.Error(), "Nothing was sent") { + t.Errorf("%s: %v", name, err) + } + } + // At the limit is taken. + if _, err := readCommitAsk(map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b", + "files": filesArg(map[string]any{"path": "x", "content_base64": base64.StdEncoding.EncodeToString([]byte(quarter))})}); err != nil { + t.Errorf("a file at the limit was refused: %v", err) + } +} + +func TestCodeSearchStopsAtItsDeadlineAndSaysSo(t *testing.T) { + gz := archive(t, map[string]string{"a.md": "forge\n", "b.md": "forge\n"}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := &Found{Matches: []Match{}} + if err := searchArchive(ctx, gz, "", regexp.MustCompile("forge"), 100, out); err != nil { + t.Fatal(err) + } + if !out.Stopped || out.Files != 0 { + t.Fatalf("a search past its deadline went on: %+v", out) + } + // Within its deadline, the same archive is searched whole. + out = &Found{Matches: []Match{}} + if err := searchArchive(context.Background(), gz, "", regexp.MustCompile("forge"), 100, out); err != nil || out.Stopped || len(out.Matches) != 2 { + t.Fatalf("%v %+v", err, out) + } +} diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go index 7cfa04f9..ee5e1e0a 100644 --- a/modules/gitea/cmd/forge/main.go +++ b/modules/gitea/cmd/forge/main.go @@ -53,7 +53,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { "independent review approved and whose required checks passed (gitea_merge_pull_request). Carries " + "either files — each {path, content (text) | content_base64 | delete: true}, created or updated as " + "the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " + - "Answers the commit it made and its parent, read back from the branch. Replaces git commit and git " + + "At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " + + "the parent the forge made it on, read back from the branch, and says when the branch moved under " + + "it. Replaces git commit and git " + "push of a feature branch. (a)", Input: map[string]any{ "type": "object", diff --git a/modules/gitea/cmd/forge/reads.go b/modules/gitea/cmd/forge/reads.go index 464a4b85..acf6e861 100644 --- a/modules/gitea/cmd/forge/reads.go +++ b/modules/gitea/cmd/forge/reads.go @@ -72,7 +72,7 @@ func history(ctx context.Context, f *Forge, owner, name, ref, path string, limit q.Set("verification", "false") q.Set("files", "false") var got []forgeCommit - if err := f.getJSON(ctx, repo+"/commits?"+q.Encode(), &got); err != nil { + if err := f.GetJSON(ctx, repo+"/commits?"+q.Encode(), &got); err != nil { return nil, err } h := &History{Repository: owner + "/" + name, Ref: ref, Path: path, Page: page, Commits: []CommitLine{}, More: len(got) >= limit} @@ -121,7 +121,7 @@ func compare(ctx context.Context, f *Forge, owner, name, base, head string) (*Co Total int `json:"total_commits"` Commits []forgeCommit `json:"commits"` } - if err := f.getJSON(ctx, repo+"/compare/"+b+"..."+h, &got); err != nil { + if err := f.GetJSON(ctx, repo+"/compare/"+b+"..."+h, &got); err != nil { return nil, err } c := &Comparison{Repository: owner + "/" + name, Base: base, Head: head, Total: got.Total, Commits: []CommitLine{}, Files: []FileLine{}} @@ -195,7 +195,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re SHA string `json:"sha"` } `json:"head"` } - if err := f.getJSON(ctx, pr, &pull); err != nil { + if err := f.GetJSON(ctx, pr, &pull); err != nil { if errors.Is(err, errNotFound) { return nil, refuse("%s/%s has no pull request %d", owner, name, number) } @@ -218,7 +218,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re var all []forgeReview for p := 1; ; p++ { var batch []forgeReview - if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews?limit=%d&page=%d", pr, page, p), &batch); err != nil { + if err := f.GetJSON(ctx, fmt.Sprintf("%s/reviews?limit=%d&page=%d", pr, page, p), &batch); err != nil { return nil, err } all = append(all, batch...) @@ -240,7 +240,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re CommitID string `json:"commit_id"` Resolver *forgeUser `json:"resolver"` } - if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews/%d/comments", pr, r.ID), &cs); err != nil { + if err := f.GetJSON(ctx, fmt.Sprintf("%s/reviews/%d/comments", pr, r.ID), &cs); err != nil { return nil, fmt.Errorf("review %d's line comments could not be read: %w", r.ID, err) } for _, c := range cs { @@ -292,7 +292,7 @@ func listDir(ctx context.Context, f *Forge, owner, name, path, ref string) (*Lis if ref != "" { at += "?ref=" + queryEscape(ref) } - body, err := f.do(ctx, "GET", at, nil, maxBody) + body, err := f.Do(ctx, "GET", at, nil, maxBody) if err != nil { if errors.Is(err, errNotFound) { return nil, refuse("%s/%s has no %q at %s", owner, name, path, orDefault(ref)) diff --git a/modules/gitea/cmd/forge/rules.go b/modules/gitea/cmd/forge/rules.go new file mode 100644 index 00000000..25f6eb70 --- /dev/null +++ b/modules/gitea/cmd/forge/rules.go @@ -0,0 +1,116 @@ +package main + +import ( + "errors" + "regexp" + "strings" +) + +// ruleCovers is whether a protection rule's name covers a branch, as the forge itself decides it (Gitea 1.27, +// models/git/protected_branch.go Match and modules/glob, novox/hq issue 539): a name without a glob character +// covers the branch of that name in any case; any other name is a glob with `/` as its separator — `*` any +// run within a segment, `**` any run across them, `?` one character of a segment, `[abc]`, `[a-z]` and `[!abc]` +// one character, `{a,b}` either, `\` the next character itself — matched case-sensitively; a glob the forge +// cannot compile covers its own text exactly. The TypeScript bundle's ruleCovers (guard.ts) is the same rule. +func ruleCovers(rule, branch string) bool { + if !strings.ContainsAny(rule, `*?\[]{}`) { + return strings.EqualFold(rule, branch) + } + re, err := globRegexp(rule) + if err != nil { + return rule == branch + } + return re.MatchString(branch) +} + +// globRegexp is the forge's glob as a regular expression, compiled as modules/glob compiles it with `/` as +// the separator. +func globRegexp(glob string) (*regexp.Regexp, error) { + g := []rune(glob) + var b strings.Builder + pos := 0 + var compile func(sub bool) error + chars := func() error { + b.WriteByte('[') + if pos < len(g) && g[pos] == '!' { + pos++ + b.WriteByte('^') + } + for pos < len(g) { + c := g[pos] + pos++ + switch { + case c == ']': + b.WriteByte(']') + return nil + case c == '\\': + if pos >= len(g) { + return errors.New("unterminated character class escape") + } + b.WriteByte('\\') + b.WriteRune(g[pos]) + pos++ + default: + b.WriteRune(c) + } + } + return errors.New("unterminated character class") + } + compile = func(sub bool) error { + for pos < len(g) { + c := g[pos] + pos++ + if sub && c == '}' { + b.WriteByte(')') + return nil + } + switch c { + case '*': + if pos < len(g) && g[pos] == '*' { + pos++ + b.WriteString(".*") + } else { + b.WriteString("[^/]*") + } + case '?': + b.WriteString("[^/]") + case '+': + b.WriteString(`\+`) + case '[': + if err := chars(); err != nil { + return err + } + case '{': + b.WriteByte('(') + if err := compile(true); err != nil { + return err + } + case ',': + if sub { + b.WriteByte('|') + } else { + b.WriteByte(',') + } + case '\\': + if pos >= len(g) { + return errors.New("no character to escape") + } + b.WriteByte('\\') + b.WriteRune(g[pos]) + pos++ + case '.', '^', '$', '(', ')', '|': + b.WriteByte('\\') + b.WriteRune(c) + default: + b.WriteRune(c) + } + } + return nil + } + b.WriteByte('^') + if err := compile(false); err != nil { + return nil, err + } + b.WriteByte('$') + return regexp.Compile(b.String()) +} diff --git a/modules/gitea/cmd/forge/search.go b/modules/gitea/cmd/forge/search.go index 3931ced2..3abad320 100644 --- a/modules/gitea/cmd/forge/search.go +++ b/modules/gitea/cmd/forge/search.go @@ -32,6 +32,7 @@ type Found struct { Files int `json:"files_searched"` Skipped []string `json:"skipped,omitempty"` Truncated bool `json:"truncated"` + Stopped bool `json:"stopped_at_deadline,omitempty"` Said []string `json:"said"` } @@ -69,7 +70,7 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { ref := a.Ref if ref == "" { var r Repo - if err := f.getJSON(ctx, repo, &r); err != nil { + if err := f.GetJSON(ctx, repo, &r); err != nil { return nil, err } if r.Empty || r.DefaultBranch == "" { @@ -87,7 +88,7 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { return nil, err } } - body, err := f.do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive) + body, err := f.Do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive) if err != nil { if errors.Is(err, errNotFound) { return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref) @@ -95,11 +96,15 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { return nil, err } out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}} - if err := searchArchive(body, prefix, re, a.Max, out); err != nil { + if err := searchArchive(ctx, body, prefix, re, a.Max, out); err != nil { return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err) } said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files) - if out.Truncated { + switch { + case out.Stopped: + said = fmt.Sprintf("the search stopped at its deadline of %s after %d files, with %d matches: the rest of the "+ + "archive was not searched. Narrow with path", deadline, out.Files, len(out.Matches)) + case out.Truncated: said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches)) } out.Said = append(out.Said, said) @@ -110,18 +115,32 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { } // searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the -// lines that match, from text files under the prefix. -func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error { +// lines that match, from text files under the prefix. It stops at the call's deadline (novox/hq issue 539): +// what it found by then is kept, and out.Stopped says the rest was not searched. +func searchArchive(ctx context.Context, gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error { zr, err := gzip.NewReader(bytes.NewReader(gz)) if err != nil { return err } - tr := tar.NewReader(zr) + tr := tar.NewReader(ctxReader{ctx, zr}) + stopped := func(err error) bool { + if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) { + out.Stopped = true + return true + } + return false + } for { + if stopped(nil) { + return nil + } h, err := tr.Next() if err == io.EOF { return nil } + if stopped(err) { + return nil + } if err != nil { return err } @@ -140,6 +159,9 @@ func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Fo continue } content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1)) + if stopped(err) { + return nil + } if err != nil { return err } @@ -169,3 +191,16 @@ func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Fo } } } + +// ctxReader is a reader that ends when its context does: decompressing is the long part of a search. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/modules/gitea/cmd/npm-registry/forge.go b/modules/gitea/cmd/npm-registry/forge.go index 948a3d21..c90daf9e 100644 --- a/modules/gitea/cmd/npm-registry/forge.go +++ b/modules/gitea/cmd/npm-registry/forge.go @@ -2,45 +2,33 @@ package main import ( "context" - "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "os" "strings" "time" + + "gitea/internal/forgehttp" ) -// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin -// account, which the vault delivered into a file this module owns (novox/hq ADR 0086). Basic -// authentication, so this bundle mints nothing and keeps nothing: the TypeScript bundle's token is its -// own. The password is read when a call is made and never logged, answered or put in an error. +// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account +// (internal/forgehttp, shared with the forge bundle beside it). Basic authentication, so this bundle mints +// nothing and keeps nothing: the TypeScript bundle's token is its own. // Forge is the forge's HTTP interface. type Forge struct { - URL string // e.g. http://127.0.0.1:3000 - User string - PasswordFile string - Owner string // the package registry's owner, as the seat serves it - HTTP *http.Client + forgehttp.Client + Owner string // the package registry's owner, as the seat serves it } // ForgeFromEnv is the forge as the manifest hands it to this bundle. func ForgeFromEnv() (*Forge, error) { - f := &Forge{ - URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), - User: os.Getenv("MESH_GITEA_ADMIN_USER"), - PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), - Owner: os.Getenv("MESH_NPM_OWNER"), - } - var missing []string - for name, v := range map[string]string{"MESH_GITEA_URL": f.URL, "MESH_GITEA_ADMIN_USER": f.User, - "MESH_GITEA_ADMIN_PASSWORD_FILE": f.PasswordFile, "MESH_NPM_OWNER": f.Owner} { - if v == "" { - missing = append(missing, name) - } + c, missing := forgehttp.FromEnv() + f := &Forge{Client: c, Owner: os.Getenv("MESH_NPM_OWNER")} + if f.Owner == "" { + missing = append(missing, "MESH_NPM_OWNER") } if len(missing) > 0 { return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest", strings.Join(missing, ", ")) @@ -49,86 +37,14 @@ func ForgeFromEnv() (*Forge, error) { } // errNotFound is the forge answering 404. -var errNotFound = errors.New("not found") - -// statusError is an answer the forge gave that is not a success, said without the request's credential. -type statusError struct { - method, path string - status int - said string -} - -func (e *statusError) Error() string { - return fmt.Sprintf("the forge answered %d to %s %s: %s", e.status, e.method, e.path, e.said) -} - -func (f *Forge) client() *http.Client { - if f.HTTP != nil { - return f.HTTP - } - return &http.Client{Timeout: 20 * time.Second} -} - -func (f *Forge) password() (string, error) { - raw, err := os.ReadFile(f.PasswordFile) - if err != nil { - // The path is the manifest's, not a secret; the content never appears. - return "", fmt.Errorf("the forge's admin password cannot be read from its file: %w", err) - } - return strings.TrimSpace(string(raw)), nil -} +var errNotFound = forgehttp.ErrNotFound // maxBody is the largest answer read from the forge. -const maxBody = 32 << 20 +const maxBody = forgehttp.MaxBody -// do sends one request and answers the body; 404 is errNotFound. +// do sends one request without a body and answers the body; 404 is errNotFound. func (f *Forge) do(ctx context.Context, method, path string) ([]byte, error) { - password, err := f.password() - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, method, f.URL+path, nil) - if err != nil { - return nil, err - } - req.SetBasicAuth(f.User, password) - req.Header.Set("Accept", "application/json") - res, err := f.client().Do(req) - if err != nil { - return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err) - } - defer res.Body.Close() - // One byte past the cap is read, so a body the cap cut is told from one that fits it exactly: a - // cut body is refused, never parsed as if it were whole. - body, err := io.ReadAll(io.LimitReader(res.Body, maxBody+1)) - if err != nil { - return nil, err - } - if len(body) > maxBody { - return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, maxBody>>20) - } - switch { - case res.StatusCode == http.StatusNotFound: - return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound) - case res.StatusCode >= 300: - said := strings.TrimSpace(string(body)) - if len(said) > 300 { - said = said[:300] + "…" - } - return nil, &statusError{method, path, res.StatusCode, said} - } - return body, nil -} - -func (f *Forge) getJSON(ctx context.Context, path string, into any) error { - body, err := f.do(ctx, http.MethodGet, path) - if err != nil { - return err - } - if err := json.Unmarshal(body, into); err != nil { - return fmt.Errorf("the forge's answer to %s is not readable: %w", path, err) - } - return nil + return f.Do(ctx, method, path, nil, maxBody) } // PackageVersion is one version of one package, as the forge lists it. @@ -146,7 +62,7 @@ func (f *Forge) Packages(ctx context.Context) ([]PackageVersion, error) { for page := 1; page <= 1000; page++ { var batch []PackageVersion path := fmt.Sprintf("/api/v1/packages/%s?type=npm&page=%d&limit=%d", url.PathEscape(f.Owner), page, pageSize) - if err := f.getJSON(ctx, path, &batch); err != nil { + if err := f.GetJSON(ctx, path, &batch); err != nil { return nil, err } all = append(all, batch...) @@ -166,7 +82,7 @@ func (f *Forge) Size(ctx context.Context, name, version string) (int64, error) { var files []struct { Size int64 `json:"size"` } - if err := f.getJSON(ctx, f.versionPath(name, version)+"/files", &files); err != nil { + if err := f.GetJSON(ctx, f.versionPath(name, version)+"/files", &files); err != nil { return 0, err } var total int64 @@ -182,7 +98,7 @@ func (f *Forge) DistTags(ctx context.Context, name string) (map[string]string, e DistTags map[string]string `json:"dist-tags"` } path := fmt.Sprintf("/api/packages/%s/npm/%s", url.PathEscape(f.Owner), url.PathEscape(name)) - if err := f.getJSON(ctx, path, &meta); err != nil { + if err := f.GetJSON(ctx, path, &meta); err != nil { return nil, err } return meta.DistTags, nil @@ -208,7 +124,7 @@ func (f *Forge) Repositories(ctx context.Context) ([]Repository, error) { var found struct { Data []Repository `json:"data"` } - if err := f.getJSON(ctx, fmt.Sprintf("/api/v1/repos/search?page=%d&limit=%d", page, pageSize), &found); err != nil { + if err := f.GetJSON(ctx, fmt.Sprintf("/api/v1/repos/search?page=%d&limit=%d", page, pageSize), &found); err != nil { return nil, err } all = append(all, found.Data...) @@ -239,7 +155,7 @@ func (f *Forge) Files(ctx context.Context, repo Repository) ([]string, error) { } path := fmt.Sprintf("/api/v1/repos/%s/git/trees/%s?recursive=true&page=%d&per_page=1000", repoPath(repo.FullName), url.PathEscape(repo.DefaultBranch), page) - if err := f.getJSON(ctx, path, &tree); err != nil { + if err := f.GetJSON(ctx, path, &tree); err != nil { return nil, err } for _, e := range tree.Tree { diff --git a/modules/gitea/cmd/npm-registry/retention_test.go b/modules/gitea/cmd/npm-registry/retention_test.go index 9c252803..03917f59 100644 --- a/modules/gitea/cmd/npm-registry/retention_test.go +++ b/modules/gitea/cmd/npm-registry/retention_test.go @@ -15,6 +15,8 @@ import ( "sync" "testing" "time" + + "gitea/internal/forgehttp" ) // fakeForge answers the forge's interface from maps, and records every delete. @@ -38,7 +40,7 @@ func (f *fakeForge) serve() *Forge { if err := os.WriteFile(file, []byte(f.password+"\n"), 0o600); err != nil { f.t.Fatal(err) } - return &Forge{URL: srv.URL, User: "admin", PasswordFile: file, Owner: "acme"} + return &Forge{Client: forgehttp.Client{URL: srv.URL, User: "admin", PasswordFile: file}, Owner: "acme"} } var ( @@ -501,7 +503,7 @@ func TestAnAnswerTheCapCutIsRefused(t *testing.T) { if err := os.WriteFile(file, []byte("x\n"), 0o600); err != nil { t.Fatal(err) } - f := &Forge{URL: srv.URL, User: "admin", PasswordFile: file, Owner: "acme"} + f := &Forge{Client: forgehttp.Client{URL: srv.URL, User: "admin", PasswordFile: file}, Owner: "acme"} if _, err := f.do(context.Background(), http.MethodGet, "/big"); err == nil || !strings.Contains(err.Error(), "not read whole") { t.Fatalf("a cut answer: %v", err) } diff --git a/modules/gitea/guard.ts b/modules/gitea/guard.ts index b0d68b0f..be34ad0d 100644 --- a/modules/gitea/guard.ts +++ b/modules/gitea/guard.ts @@ -164,21 +164,89 @@ export function judgeApiCall(method: string, rawPath: string, body: unknown): Ap }; } -/** Whether a protection rule's name — a branch, or a glob of them (`*`, `**`) — covers a branch. */ +/** + * Whether a protection rule's name covers a branch, as the forge itself decides it (Gitea 1.27, + * models/git/protected_branch.go Match and modules/glob; novox/hq issue 539): a name without a glob character + * covers the branch of that name in any case; any other name is a glob with `/` as its separator — `*` any run + * within a segment, `**` any run across them, `?` one character of a segment, `[abc]`, `[a-z]` and `[!abc]` one + * character, `{a,b}` either, `\` the next character itself — matched case-sensitively; a glob the forge cannot + * compile covers its own text exactly. The forge bundle's ruleCovers (cmd/forge/rules.go) is the same rule. + */ export function ruleCovers(rule: string, branch: string): boolean { - if (rule === branch) return true; - if (!/[*?[]/.test(rule)) return false; - let re = ""; - for (let i = 0; i < rule.length; i++) { - const c = rule[i]; - if (c === "*" && rule[i + 1] === "*") { - re += ".*"; - i++; - } else if (c === "*") re += "[^/]*"; - else if (c === "?") re += "[^/]"; - else re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + if (!/[*?\\[\]{}]/.test(rule)) return rule.toLowerCase() === branch.toLowerCase(); + const re = globRegExp(rule); + return re ? re.test(branch) : rule === branch; +} + +/** The forge's glob as a regular expression, compiled as its modules/glob compiles it with `/` as separator. */ +function globRegExp(glob: string): RegExp | null { + const g = Array.from(glob); + let out = ""; + let pos = 0; + const lit = (c: string) => c.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); + const chars = (): boolean => { + out += "["; + if (g[pos] === "!") { + pos++; + out += "^"; + } + while (pos < g.length) { + const c = g[pos++]; + if (c === "]") { + out += "]"; + return true; + } + if (c === "\\") { + if (pos >= g.length) return false; + out += "\\" + g[pos++]; + } else out += c === "[" ? "\\[" : c; + } + return false; + }; + const compile = (sub: boolean): boolean => { + while (pos < g.length) { + const c = g[pos++]; + if (sub && c === "}") { + out += ")"; + return true; + } + switch (c) { + case "*": + if (g[pos] === "*") { + pos++; + out += ".*"; + } else out += "[^/]*"; + break; + case "?": + out += "[^/]"; + break; + case "[": + if (!chars()) return false; + break; + case "{": + out += "("; + if (!compile(true)) return false; + break; + case ",": + out += sub ? "|" : ","; + break; + case "\\": + if (pos >= g.length) return false; + out += lit(g[pos++]); + break; + default: + out += lit(c); + } + } + // An alternative left open is a glob the forge cannot compile. + return !sub; + }; + if (!compile(false)) return null; + try { + return new RegExp(`^${out}$`, "u"); + } catch { + return null; } - return new RegExp(`^${re}$`).test(branch); } /** Refuses a file written to the default branch or to a branch a protection rule names. */ diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go new file mode 100644 index 00000000..9b76e553 --- /dev/null +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -0,0 +1,151 @@ +// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, +// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered +// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call +// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539). +package forgehttp + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +// MaxBody is the largest JSON answer read from the forge. +const MaxBody = 32 << 20 + +// Client is the forge's HTTP interface. +type Client struct { + URL string // e.g. http://127.0.0.1:3000 + User string + PasswordFile string + HTTP *http.Client // when nil, a client with Timeout + Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero +} + +// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. +func FromEnv() (Client, []string) { + c := Client{ + URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), + User: os.Getenv("MESH_GITEA_ADMIN_USER"), + PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), + } + var missing []string + for _, kv := range [][2]string{{"MESH_GITEA_URL", c.URL}, {"MESH_GITEA_ADMIN_USER", c.User}, + {"MESH_GITEA_ADMIN_PASSWORD_FILE", c.PasswordFile}} { + if kv[1] == "" { + missing = append(missing, kv[0]) + } + } + return c, missing +} + +// ErrNotFound is the forge answering 404. +var ErrNotFound = errors.New("not found") + +// StatusError is an answer the forge gave that is not a success, said without the request's credential. +type StatusError struct { + Method, Path string + Status int + Said string +} + +func (e *StatusError) Error() string { + return fmt.Sprintf("the forge answered %d to %s %s: %s", e.Status, e.Method, e.Path, e.Said) +} + +func (c *Client) client() *http.Client { + if c.HTTP != nil { + return c.HTTP + } + timeout := c.Timeout + if timeout <= 0 { + timeout = 20 * time.Second + } + return &http.Client{Timeout: timeout} +} + +func (c *Client) password() (string, error) { + raw, err := os.ReadFile(c.PasswordFile) + if err != nil { + // The path is the manifest's, not a secret; the content never appears. + return "", fmt.Errorf("the forge's admin password cannot be read from its file: %w", err) + } + return strings.TrimSpace(string(raw)), nil +} + +// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit; +// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it +// exactly: a cut body is refused, never read as if it were whole. +func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { + password, err := c.password() + if err != nil { + return nil, err + } + var body io.Reader + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + return nil, err + } + body = bytes.NewReader(raw) + } + req, err := http.NewRequestWithContext(ctx, method, c.URL+path, body) + if err != nil { + return nil, err + } + req.SetBasicAuth(c.User, password) + req.Header.Set("Accept", "application/json") + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + res, err := c.client().Do(req) + if err != nil { + return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err) + } + defer res.Body.Close() + got, err := io.ReadAll(io.LimitReader(res.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(got)) > limit { + return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) + } + switch { + case res.StatusCode == http.StatusNotFound: + return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound) + case res.StatusCode >= 300: + said := strings.TrimSpace(string(got)) + if len(said) > 300 { + said = said[:300] + "…" + } + return nil, &StatusError{method, path, res.StatusCode, said} + } + return got, nil +} + +// GetJSON reads one answer into into. +func (c *Client) GetJSON(ctx context.Context, path string, into any) error { + return c.SendJSON(ctx, http.MethodGet, path, nil, into) +} + +// SendJSON sends one request and reads its answer into into, when into is not nil. +func (c *Client) SendJSON(ctx context.Context, method, path string, payload, into any) error { + body, err := c.Do(ctx, method, path, payload, MaxBody) + if err != nil { + return err + } + if into == nil { + return nil + } + if err := json.Unmarshal(body, into); err != nil { + return fmt.Errorf("the forge's answer to %s %s is not readable: %w", method, path, err) + } + return nil +} diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index a9d84830..8c08a6a3 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -146,6 +146,27 @@ test("gitea_api refuses a file written to the default branch or one a rule guard assert.deepEqual(writes(), ["PUT /repos/novox/mesh-catalog/contents/modules/sudo/module.json"]); }); +test("a protection rule covers a branch as the forge decides it: classes, alternatives, a plain name in any case (issue 539)", async () => { + const cases: Array<[string, string, boolean]> = [ + ["main", "main", true], ["main", "mainx", false], ["release/*", "release/1.0", true], ["release/*", "release/a/b", false], + ["release/**", "release/a/b", true], ["v?", "v1", true], ["a.b", "axb", false], + ["release-[ab]", "release-a", true], ["release-[ab]", "release-c", false], ["release-[!ab]", "release-c", true], + ["v[0-9].x", "v7.x", true], ["v[0-9].x", "vA.x", false], ["{main,master}", "master", true], + ["{main,master}", "mainline", false], ["rel/{a,b}/*", "rel/b/1", true], ["rel/{a,b}/*", "rel/c/1", false], + ["Main", "main", true], ["main", "MAIN", true], ["Feat/*", "feat/x", false], ["a\\*", "a*", true], + ["a\\*", "ab", false], ["a+b", "a+b", true], ["a+b", "aab", false], ["{open", "{open", true], + ["[open", "[open", true], ["[open", "o", false], + ]; + for (const [rule, branch, want] of cases) assert.equal(ruleCovers(rule, branch), want, `${rule} covers ${branch}`); + const put = (branch: string) => tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/contents/README.md", + method: "PUT", body: { content: "e30=", sha: "x", branch } }); + state.rules.push({ rule_name: "release-[ab]" }, { rule_name: "{stable,lts}/*" }); + await assert.rejects(put("release-a"), /protection rule release-\[ab\]/); + await assert.rejects(put("lts/2"), /protection rule \{stable,lts\}\/\*/); + await assert.rejects(put("MAIN"), /protection rule main/); + assert.deepEqual(writes(), []); +}); + test("gitea_api still writes what an author writes", async () => { await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });