diff --git a/modules/gitea/README.md b/modules/gitea/README.md index 557e673f..ab42d456 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -2,7 +2,7 @@ The forge, and the mesh's npm package registry: it claims the `git` and `npm-package-registry` seats. Its tools, its events and the provisioner of registry accounts are the TypeScript bundle (`index.ts`, -`tools/`, `provisioner/`). This README covers the Go bundle beside it. +`tools/`, `provisioner/`). This README covers the two Go bundles beside it. ## Announcing pull requests to the merge check (novox/hq to-be 45 §9, issue 347) @@ -28,6 +28,46 @@ Every new head of an open pull request is announced as `pull.updated`, once, and - **A pull request or issue number that is not one is refused** before the forge is asked: a tool called with a misnamed argument sent `pulls/NaN`. +## An agent's commit and the reads around it (novox/hq issue 375) + +An agent that finished a change on a branch had no way to put it on the forge: the module had no commit or +push tool, `gitea_api` refused the forge's patch endpoint, and its only file write took one whole file +base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an agent otherwise made through +`gitea_api` as raw JSON, summarised. + +| tool | | what | +|---|---|---| +| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) or `patch` (a unified diff, as `git diff` writes it); answers the commit and its parent | +| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject | +| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch | +| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments | +| `gitea_list_dir` | r | a directory's entries at a ref: name, type, a file's size | +| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path | + +**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266): +never the repository's default branch, never a branch a protection rule names (a glob rule covers by the same +matcher as `guard.ts`). The rules are read before anything is sent; rules that cannot be read refuse the +commit. Nothing about merging changes: a merge still needs an independent review at the head and both +required checks green, through `gitea_merge_pull_request`. + +**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the +ref holds is updated, one it does not is created, and an update or delete carries the blob read just +before, so a file someone changed meanwhile is refused by the forge rather than overwritten. A `patch` goes +to the forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than +the file. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the +local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a +branch not at the commit the forge named is a failure, never a success. + +**Code search** has no endpoint on the forge, so the ref's archive (`/archive/{ref}.tar.gz`, at most 64 MiB) +is read once and searched in memory; a binary file is passed over, a file over 4 MiB is named under +`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. + +It reaches the forge as the npm-registry bundle does (below), and one tool call takes at most 50 seconds. +`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on +a new branch from its base; an existing branch committed on; the default branch, a protected branch and +unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another +commit failing; every malformed ask refused; and each read summarised. + ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) Every publish adds a version to the registry, and until this nothing removed one. The Go bundle diff --git a/modules/gitea/cmd/forge/commit.go b/modules/gitea/cmd/forge/commit.go new file mode 100644 index 00000000..f60c33b6 --- /dev/null +++ b/modules/gitea/cmd/forge/commit.go @@ -0,0 +1,332 @@ +package main + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "strings" + "unicode/utf8" +) + +// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull +// request / approval rule without anyone pushing by hand. By the rule gitea_api already enforces for a file +// written (novox/hq ADR 0266): never the repository's default branch, never a branch a protection rule names. +// Nothing about merging changes — that still needs a review at the head and both required checks green +// (gitea_merge_pull_request). + +// FileChange is one file of a commit: its new content, as text or base64, or its deletion. +type FileChange struct { + Path string + Content *string + ContentBase64 *string + Delete bool +} + +// CommitAsk is one gitea_commit call. +type CommitAsk struct { + Owner, Repo string + Branch string + Base string // the branch or commit a new branch starts from; the default branch when empty + Message string + Files []FileChange + Patch string // a unified diff, as git diff writes it, instead of Files +} + +// Committed is what gitea_commit made. +type Committed struct { + Repository string `json:"repository"` + Branch string `json:"branch"` + Created bool `json:"created"` + Base string `json:"base,omitempty"` + Parent string `json:"parent"` + Commit string `json:"commit"` + Files []CommittedFile `json:"files,omitempty"` + Said []string `json:"said"` +} + +// CommittedFile is one file the commit changed, and how. +type CommittedFile struct { + Path string `json:"path"` + Operation string `json:"operation"` +} + +// maxFiles is the most files one commit carries. +const maxFiles = 200 + +// readCommitAsk reads the tool's arguments. +func readCommitAsk(args map[string]any) (CommitAsk, error) { + a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"), + Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))} + a.Patch = raw(args, "patch") + if a.Branch == "" { + return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent") + } + if a.Message == "" { + return a, refuse("message is required: the commit's message. Nothing was sent") + } + files, given := args["files"] + if given && files != nil { + list, ok := files.([]any) + if !ok { + return a, refuse("files is a list of {path, content | content_base64 | delete: true}. Nothing was sent") + } + for i, item := range list { + m, ok := item.(map[string]any) + if !ok { + return a, refuse("files[%d] is not an object {path, content | content_base64 | delete: true}. Nothing was sent", i) + } + fc := FileChange{Path: text(m, "path")} + if v, ok := m["content"]; ok && v != nil { + s, ok := v.(string) + if !ok { + return a, refuse("files[%d].content is text. Nothing was sent", i) + } + fc.Content = &s + } + if v, ok := m["content_base64"]; ok && v != nil { + s, ok := v.(string) + if !ok { + return a, refuse("files[%d].content_base64 is text. Nothing was sent", i) + } + fc.ContentBase64 = &s + } + if v, ok := m["delete"]; ok && v != nil { + b, ok := v.(bool) + if !ok { + return a, refuse("files[%d].delete is true or false. Nothing was sent", i) + } + fc.Delete = b + } + a.Files = append(a.Files, fc) + } + } + switch { + case len(a.Files) == 0 && strings.TrimSpace(a.Patch) == "": + return a, refuse("give files (each {path, content | content_base64 | delete: true}) or patch (a unified diff, as git diff writes it). Nothing was sent") + case len(a.Files) > 0 && strings.TrimSpace(a.Patch) != "": + return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent") + case len(a.Files) > maxFiles: + return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles) + } + seen := map[string]bool{} + for i, fc := range a.Files { + if strings.HasPrefix(fc.Path, "/") { + return a, refuse("files[%d].path %q is relative to the repository's root, without a leading slash. Nothing was sent", i, fc.Path) + } + if _, err := segments("file path", fc.Path); err != nil { + return a, err + } + if seen[fc.Path] { + return a, refuse("%s is named twice in files. Nothing was sent", fc.Path) + } + seen[fc.Path] = true + given := 0 + for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} { + if g { + given++ + } + } + if given != 1 { + return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path) + } + if fc.Content != nil && !utf8.ValidString(*fc.Content) { + return a, refuse("files[%d].content (%s) is not UTF-8 text: give it as content_base64. Nothing was sent", i, fc.Path) + } + if fc.ContentBase64 != nil { + if _, err := base64.StdEncoding.DecodeString(*fc.ContentBase64); err != nil { + return a, refuse("files[%d].content_base64 (%s) is not base64: %v. Nothing was sent", i, fc.Path, err) + } + } + } + return a, nil +} + +// targetRefused says why a commit may not go on a branch: the default branch, or one a protection rule names. +func targetRefused(target, defaultBranch string, rules []Protection) error { + if target == defaultBranch { + return refuse("a commit on %s, the repository's default branch, is refused: a change reaches it by a pull request "+ + "that passes its checks (novox/hq ADR 0266). Put it on a branch of its own. Nothing was sent", target) + } + var guarding []string + for _, r := range rules { + if n := r.name(); n != "" && ruleCovers(n, target) { + guarding = append(guarding, n) + } + } + if len(guarding) > 0 { + return refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+ + "branch of its own. Nothing was sent", target, strings.Join(guarding, ", ")) + } + return nil +} + +// commit puts one commit on the asked branch, after the judgement, and reads the branch back. +func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) { + repo, err := repoPath(a.Owner, a.Repo) + if err != nil { + return nil, err + } + if _, err := segments("branch", a.Branch); err != nil { + return nil, err + } + var r Repo + if err := f.getJSON(ctx, repo, &r); err != nil { + return nil, err + } + if r.DefaultBranch == "" { + return nil, fmt.Errorf("the forge did not say %s/%s's default branch, so whether %s is it cannot be judged. Nothing was sent", a.Owner, a.Repo, a.Branch) + } + // Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed). + var rules []Protection + if err := f.getJSON(ctx, repo+"/branch_protections", &rules); err != nil { + return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) + } + if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil { + return nil, err + } + + out := &Committed{Repository: a.Owner + "/" + a.Repo, Branch: a.Branch} + existing, err := f.branch(ctx, repo, a.Branch) + if err != nil { + return nil, err + } + ref := a.Branch // where the files are read and the commit goes + if existing != nil { + out.Parent = existing.Commit.ID + if a.Base != "" { + out.Said = append(out.Said, fmt.Sprintf("%s exists, at %s: base %s was not used", a.Branch, short(existing.Commit.ID), a.Base)) + } + } else { + out.Created = true + out.Base = a.Base + if out.Base == "" { + out.Base = r.DefaultBranch + } + ref = out.Base + b, err := f.branch(ctx, repo, out.Base) + if err != nil { + return nil, err + } + if b == nil { + return nil, refuse("%s does not exist, and its base %s is not a branch of %s/%s. Nothing was sent", a.Branch, out.Base, a.Owner, a.Repo) + } + out.Parent = b.Commit.ID + // The files are judged at the commit just read; a file the base changes meanwhile is refused by the + // forge, whose update and delete carry the blob read here. + ref = b.Commit.ID + } + + // Where the forge takes the commit: on the branch, or on a new one from the base's commit. + on := map[string]any{"message": a.Message} + if out.Created { + on["branch"] = out.Base + on["new_branch"] = a.Branch + } else { + on["branch"] = a.Branch + } + + var made struct { + Commit struct { + SHA string `json:"sha"` + } `json:"commit"` + } + if a.Patch != "" { + on["content"] = a.Patch + if err := f.sendJSON(ctx, http.MethodPost, repo+"/diffpatch", on, &made); err != nil { + return nil, fmt.Errorf("the forge did not take the patch: %w", err) + } + } else { + var ops []map[string]any + for _, fc := range a.Files { + sha, err := f.blobAt(ctx, repo, fc.Path, ref) + if err != nil { + return nil, err + } + op := map[string]any{"path": fc.Path} + switch { + case fc.Delete && sha == "": + return nil, refuse("%s is to be deleted, but %s has no such file at %s. Nothing was sent", fc.Path, a.Owner+"/"+a.Repo, short(ref)) + case fc.Delete: + op["operation"], op["sha"] = "delete", sha + case sha == "": + op["operation"] = "create" + default: + op["operation"], op["sha"] = "update", sha + } + if !fc.Delete { + if fc.Content != nil { + op["content"] = base64.StdEncoding.EncodeToString([]byte(*fc.Content)) + } else { + op["content"] = *fc.ContentBase64 + } + } + ops = append(ops, op) + out.Files = append(out.Files, CommittedFile{Path: fc.Path, Operation: op["operation"].(string)}) + } + on["files"] = ops + if err := f.sendJSON(ctx, http.MethodPost, repo+"/contents", on, &made); err != nil { + return nil, fmt.Errorf("the forge did not take the commit: %w", err) + } + } + if made.Commit.SHA == "" { + return nil, fmt.Errorf("the forge answered the commit on %s without saying the commit it made: read the branch (gitea_list_branches) before trying again", a.Branch) + } + out.Commit = made.Commit.SHA + + // Evidence over assertion: the branch is read back, and it must be at the commit the forge said. + after, err := f.branch(ctx, repo, a.Branch) + switch { + case err != nil: + return nil, fmt.Errorf("the forge made %s on %s, but the branch could not be read back: %w", out.Commit, a.Branch, err) + case after == nil: + return nil, fmt.Errorf("the forge said it made %s on %s, but the branch does not exist", out.Commit, a.Branch) + case after.Commit.ID != out.Commit: + return nil, fmt.Errorf("the forge said it made %s on %s, but the branch is at %s: something else moved it", out.Commit, a.Branch, after.Commit.ID) + } + what := "on" + if out.Created { + what = "on the new branch" + } + out.Said = append(out.Said, fmt.Sprintf("%s committed %s %s of %s, after %s; the branch is there now. A merge "+ + "still needs a review at this head and both required checks green", short(out.Commit), what, a.Branch, out.Repository, short(out.Parent))) + return out, nil +} + +// blobAt is the blob sha of a file at a ref, "" when there is no file there. +func (f *Forge) blobAt(ctx context.Context, repo, path, ref string) (string, error) { + seg, err := segments("file path", path) + if err != nil { + return "", err + } + var entry struct { + Type string `json:"type"` + SHA string `json:"sha"` + } + var list []any + body, err := f.do(ctx, http.MethodGet, repo+"/contents/"+seg+"?ref="+queryEscape(ref), nil, maxBody) + if err != nil { + if errors.Is(err, errNotFound) { + return "", nil + } + return "", err + } + if jsonInto(body, &list) == nil { + return "", refuse("%s is a directory, not a file. Nothing was sent", path) + } + if err := jsonInto(body, &entry); err != nil { + return "", fmt.Errorf("the forge's answer about %s is not readable: %w", path, err) + } + if entry.Type != "file" && entry.Type != "symlink" { + return "", refuse("%s is a %s, not a file. Nothing was sent", path, entry.Type) + } + return entry.SHA, nil +} + +func short(sha string) string { + if len(sha) > 12 { + return sha[:12] + } + return sha +} diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go new file mode 100644 index 00000000..940e08a0 --- /dev/null +++ b/modules/gitea/cmd/forge/forge.go @@ -0,0 +1,259 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" +) + +// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account, +// which the vault delivered into a file this module owns (novox/hq ADR 0086), by basic authentication — the +// same way the npm-registry bundle beside it does. The password is read when a call is made and never logged, +// answered or put in an error. + +// Forge is the forge's HTTP interface. +type Forge struct { + URL string // e.g. http://127.0.0.1:3000 + User string + PasswordFile string + HTTP *http.Client +} + +// ForgeFromEnv is the forge as the manifest hands it to this bundle. +func ForgeFromEnv() (*Forge, error) { + f := &Forge{ + URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), + User: os.Getenv("MESH_GITEA_ADMIN_USER"), + PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), + } + var missing []string + for _, kv := range [][2]string{{"MESH_GITEA_URL", f.URL}, {"MESH_GITEA_ADMIN_USER", f.User}, + {"MESH_GITEA_ADMIN_PASSWORD_FILE", f.PasswordFile}} { + if kv[1] == "" { + missing = append(missing, kv[0]) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest", strings.Join(missing, ", ")) + } + return f, nil +} + +// errNotFound is the forge answering 404. +var errNotFound = errors.New("not found") + +// statusError is an answer the forge gave that is not a success, said without the request's credential. +type statusError struct { + method, path string + status int + said string +} + +func (e *statusError) Error() string { + return fmt.Sprintf("the forge answered %d to %s %s: %s", e.status, e.method, e.path, e.said) +} + +func (f *Forge) client() *http.Client { + if f.HTTP != nil { + return f.HTTP + } + return &http.Client{Timeout: 50 * time.Second} +} + +func (f *Forge) password() (string, error) { + raw, err := os.ReadFile(f.PasswordFile) + if err != nil { + // The path is the manifest's, not a secret; the content never appears. + return "", fmt.Errorf("the forge's admin password cannot be read from its file: %w", err) + } + return strings.TrimSpace(string(raw)), nil +} + +// maxBody is the largest JSON answer read from the forge; maxArchive the largest archive code search reads. +const ( + maxBody = 32 << 20 + maxArchive = 64 << 20 +) + +// do sends one request and answers the body, read up to limit; 404 is errNotFound. A body the limit cut is +// refused, never read as if it were whole. +func (f *Forge) do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { + password, err := f.password() + if err != nil { + return nil, err + } + var body io.Reader + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + return nil, err + } + body = bytes.NewReader(raw) + } + req, err := http.NewRequestWithContext(ctx, method, f.URL+path, body) + if err != nil { + return nil, err + } + req.SetBasicAuth(f.User, password) + req.Header.Set("Accept", "application/json") + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + res, err := f.client().Do(req) + if err != nil { + return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err) + } + defer res.Body.Close() + got, err := io.ReadAll(io.LimitReader(res.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(got)) > limit { + return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) + } + switch { + case res.StatusCode == http.StatusNotFound: + return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound) + case res.StatusCode >= 300: + said := strings.TrimSpace(string(got)) + if len(said) > 300 { + said = said[:300] + "…" + } + return nil, &statusError{method, path, res.StatusCode, said} + } + return got, nil +} + +func (f *Forge) getJSON(ctx context.Context, path string, into any) error { + return f.sendJSON(ctx, http.MethodGet, path, nil, into) +} + +func (f *Forge) sendJSON(ctx context.Context, method, path string, payload, into any) error { + body, err := f.do(ctx, method, path, payload, maxBody) + if err != nil { + return err + } + if into == nil { + return nil + } + if err := json.Unmarshal(body, into); err != nil { + return fmt.Errorf("the forge's answer to %s %s is not readable: %w", method, path, err) + } + return nil +} + +// Refused is a refusal of policy or of an argument, in the words the caller sees. +type Refused struct{ msg string } + +func (r *Refused) Error() string { return r.msg } + +func refuse(format string, a ...any) error { return &Refused{fmt.Sprintf(format, a...)} } + +var nameShape = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) + +// repoPath is /api/v1/repos/{owner}/{repo} for names that cannot become another path. +func repoPath(owner, repo string) (string, error) { + for _, v := range [][2]string{{"owner", owner}, {"name", repo}} { + if !nameShape.MatchString(v[1]) || v[1] == "." || v[1] == ".." { + return "", refuse("%q is not a repository %s", v[1], v[0]) + } + } + return "/api/v1/repos/" + owner + "/" + repo, nil +} + +// segments encodes a value that is several path segments — a branch like fix/1-x, a file's path — refusing +// any segment that could make the request another path: empty, `.` or `..`, or holding `?`, `#`, `%`, `\` +// or a control character (the same rule as the TypeScript bundle's pathSegments, guard.ts). +func segments(what, value string) (string, error) { + parts := strings.Split(value, "/") + for i, p := range parts { + if p == "" || p == "." || p == ".." || strings.ContainsAny(p, "?#%\\") || strings.IndexFunc(p, func(r rune) bool { return r < 0x20 || r == 0x7f }) >= 0 { + return "", refuse("%q is not a %s this tool puts in a path. Nothing was sent", value, what) + } + parts[i] = url.PathEscape(p) + } + return strings.Join(parts, "/"), nil +} + +// Repo is what the tools read of a repository. +type Repo struct { + FullName string `json:"full_name"` + DefaultBranch string `json:"default_branch"` + Empty bool `json:"empty"` +} + +// Protection is one branch protection rule, as far as covering a branch goes. +type Protection struct { + RuleName string `json:"rule_name"` + BranchName string `json:"branch_name"` +} + +func (p Protection) name() string { + if p.RuleName != "" { + return p.RuleName + } + return p.BranchName +} + +// ruleCovers is whether a protection rule's name — a branch, or a glob of them (`*`, `**`, `?`) — covers a +// branch: the TypeScript bundle's ruleCovers (guard.ts), which gitea_api's file writes are judged by. +func ruleCovers(rule, branch string) bool { + if rule == branch { + return true + } + if !strings.ContainsAny(rule, "*?[") { + return false + } + var re strings.Builder + re.WriteString("^") + for i := 0; i < len(rule); i++ { + c := rule[i] + switch { + case c == '*' && i+1 < len(rule) && rule[i+1] == '*': + re.WriteString(".*") + i++ + case c == '*': + re.WriteString("[^/]*") + case c == '?': + re.WriteString("[^/]") + default: + re.WriteString(regexp.QuoteMeta(string(c))) + } + } + re.WriteString("$") + ok, err := regexp.MatchString(re.String(), branch) + return err == nil && ok +} + +// Branch is one branch with the commit it points at. +type Branch struct { + Name string `json:"name"` + Commit struct { + ID string `json:"id"` + } `json:"commit"` +} + +// branch is the named branch, nil when the forge has none of that name. +func (f *Forge) branch(ctx context.Context, repo, name string) (*Branch, error) { + seg, err := segments("branch", name) + if err != nil { + return nil, err + } + var b Branch + if err := f.getJSON(ctx, repo+"/branches/"+seg, &b); err != nil { + if errors.Is(err, errNotFound) { + return nil, nil + } + return nil, err + } + return &b, nil +} diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go new file mode 100644 index 00000000..5e5cd039 --- /dev/null +++ b/modules/gitea/cmd/forge/forge_test.go @@ -0,0 +1,583 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// fakeForge is the forge's HTTP interface as far as these tools use it. +type fakeForge struct { + t *testing.T + mu sync.Mutex + password string + branches map[string]string // branch -> commit + trees map[string]map[string]string // commit -> path -> content + rules []Protection + rulesBroken bool + staleRead bool // the branch is not moved by a write: a forge that says what it did not do + writes []map[string]any + patches []map[string]any + next int + archive []byte + pullHead string + reviews []map[string]any + comments map[string][]map[string]any + commits []map[string]any + dir []map[string]any +} + +func newFake(t *testing.T) *fakeForge { + return &fakeForge{ + t: t, + password: "s3cret-never-shown", + branches: map[string]string{"main": "c0", "fix/1-old": "c1"}, + trees: map[string]map[string]string{ + "c0": {"README.md": "hello\n", "docs/a.md": "a\n"}, + "c1": {"README.md": "hello\n", "docs/a.md": "a on fix\n"}, + }, + rules: []Protection{{RuleName: "main"}, {RuleName: "release/*"}}, + } +} + +func blob(content string) string { return fmt.Sprintf("%x", sha1.Sum([]byte(content))) } + +func (f *fakeForge) serve() *Forge { + dir := f.t.TempDir() + pw := filepath.Join(dir, "admin.secret") + if err := os.WriteFile(pw, []byte("s3cret-never-shown\n"), 0o600); err != nil { + f.t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(f.handle)) + f.t.Cleanup(srv.Close) + return &Forge{URL: srv.URL, User: "mesh-admin", PasswordFile: pw} +} + +func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if u, p, ok := r.BasicAuth(); !ok || u != "mesh-admin" || p != f.password { + http.Error(w, "unauthorized", 401) + return + } + reply := func(v any) { _ = json.NewEncoder(w).Encode(v) } + const repo = "/api/v1/repos/acme/app" + path := r.URL.EscapedPath() + switch { + case r.Method == "GET" && path == repo: + reply(map[string]any{"full_name": "acme/app", "default_branch": "main"}) + case r.Method == "GET" && path == repo+"/branch_protections": + if f.rulesBroken { + http.Error(w, "boom", 500) + return + } + reply(f.rules) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/branches/"): + name := strings.TrimPrefix(r.URL.Path, repo+"/branches/") + c, ok := f.branches[name] + if !ok { + http.NotFound(w, r) + return + } + reply(map[string]any{"name": name, "commit": map[string]any{"id": c}}) + case r.Method == "GET" && (path == repo+"/contents" || path == repo+"/contents/docs"): + reply(f.dir) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/contents/"): + p := strings.TrimPrefix(r.URL.Path, repo+"/contents/") + ref := r.URL.Query().Get("ref") + if c, isBranch := f.branches[ref]; isBranch { + ref = c + } + content, ok := f.trees[ref][p] + if !ok { + http.NotFound(w, r) + return + } + reply(map[string]any{"type": "file", "path": p, "sha": blob(content)}) + case r.Method == "POST" && path == repo+"/contents": + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + f.writes = append(f.writes, body) + from := f.branches[body["branch"].(string)] + tree := map[string]string{} + for k, v := range f.trees[from] { + tree[k] = v + } + for _, raw := range body["files"].([]any) { + op := raw.(map[string]any) + p := op["path"].(string) + old, had := tree[p] + switch op["operation"] { + case "create": + if had { + http.Error(w, "exists", 422) + return + } + case "update", "delete": + if !had || op["sha"] != blob(old) { + http.Error(w, "sha does not match", 409) + return + } + } + if op["operation"] == "delete" { + delete(tree, p) + continue + } + c, _ := base64.StdEncoding.DecodeString(op["content"].(string)) + tree[p] = string(c) + } + f.next++ + made := fmt.Sprintf("n%d", f.next) + f.trees[made] = tree + target := body["branch"].(string) + if nb, ok := body["new_branch"].(string); ok && nb != "" { + target = nb + } + if !f.staleRead { + f.branches[target] = made + } + reply(map[string]any{"commit": map[string]any{"sha": made}}) + case r.Method == "POST" && path == repo+"/diffpatch": + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + f.patches = append(f.patches, body) + f.next++ + made := fmt.Sprintf("p%d", f.next) + target := body["branch"].(string) + if nb, ok := body["new_branch"].(string); ok && nb != "" { + target = nb + } + f.branches[target] = made + reply(map[string]any{"commit": map[string]any{"sha": made}}) + case r.Method == "GET" && path == repo+"/commits": + f.writes = append(f.writes, map[string]any{"query": r.URL.RawQuery}) + reply(f.commits) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/compare/"): + reply(map[string]any{"total_commits": len(f.commits), "commits": f.commits}) + case r.Method == "GET" && path == repo+"/pulls/7": + reply(map[string]any{"head": map[string]any{"sha": f.pullHead}}) + case r.Method == "GET" && path == repo+"/pulls/7/reviews": + reply(f.reviews) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/pulls/7/reviews/") && strings.HasSuffix(path, "/comments"): + id := strings.TrimSuffix(strings.TrimPrefix(path, repo+"/pulls/7/reviews/"), "/comments") + reply(f.comments[id]) + case r.Method == "GET" && strings.HasPrefix(path, repo+"/archive/"): + _, _ = w.Write(f.archive) + default: + http.NotFound(w, r) + } +} + +func filesArg(files ...map[string]any) []any { + out := make([]any, len(files)) + for i, f := range files { + out[i] = f + } + return out +} + +func ask(t *testing.T, args map[string]any) CommitAsk { + t.Helper() + base := map[string]any{"owner": "acme", "repo": "app", "message": "Say why"} + for k, v := range args { + base[k] = v + } + a, err := readCommitAsk(base) + if err != nil { + t.Fatalf("ask refused: %v", err) + } + return a +} + +func TestACommitOnANewBranchCreatesUpdatesAndDeletesInOneCommitFromTheBase(t *testing.T) { + f := newFake(t) + a := ask(t, map[string]any{"branch": "fix/375-x", "files": filesArg( + map[string]any{"path": "new.md", "content": "new\n"}, + map[string]any{"path": "README.md", "content": "hello again\n"}, + map[string]any{"path": "docs/a.md", "delete": true}, + )}) + got, err := commit(context.Background(), f.serve(), a) + if err != nil { + t.Fatal(err) + } + if !got.Created || got.Base != "main" || got.Parent != "c0" || got.Commit != "n1" || f.branches["fix/375-x"] != "n1" { + t.Fatalf("%+v; branches %v", got, f.branches) + } + if len(f.writes) != 1 { + t.Fatalf("%d writes", len(f.writes)) + } + w := f.writes[0] + if w["branch"] != "main" || w["new_branch"] != "fix/375-x" || w["message"] != "Say why" { + t.Fatalf("sent %v", w) + } + ops := map[string]string{} + for _, raw := range w["files"].([]any) { + op := raw.(map[string]any) + ops[op["path"].(string)] = op["operation"].(string) + if op["operation"] == "create" && op["sha"] != nil { + t.Error("a created file carried a sha") + } + } + if ops["new.md"] != "create" || ops["README.md"] != "update" || ops["docs/a.md"] != "delete" { + t.Fatalf("ops %v", ops) + } + tree := f.trees["n1"] + if tree["README.md"] != "hello again\n" || tree["new.md"] != "new\n" || tree["docs/a.md"] != "" { + t.Fatalf("tree %v", tree) + } + if f.branches["main"] != "c0" { + t.Fatal("main moved") + } +} + +func TestACommitOnAnExistingBranchGoesOnItAndReadsItsFilesThere(t *testing.T) { + f := newFake(t) + a := ask(t, map[string]any{"branch": "fix/1-old", "base": "main", + "files": filesArg(map[string]any{"path": "docs/a.md", "content": "a fixed\n"})}) + got, err := commit(context.Background(), f.serve(), a) + if err != nil { + t.Fatal(err) + } + if got.Created || got.Parent != "c1" || f.branches["fix/1-old"] != got.Commit { + t.Fatalf("%+v", got) + } + if _, has := f.writes[0]["new_branch"]; has || f.writes[0]["branch"] != "fix/1-old" { + t.Fatalf("sent %v", f.writes[0]) + } + if !strings.Contains(strings.Join(got.Said, " "), "base main was not used") { + t.Errorf("an unused base was not said: %v", got.Said) + } +} + +func TestTheDefaultBranchAndProtectedBranchesAreRefusedBeforeAnythingIsSent(t *testing.T) { + for _, branch := range []string{"main", "release/1.0"} { + f := newFake(t) + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": branch, + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") { + t.Errorf("%s: %v", branch, err) + } + if len(f.writes)+len(f.patches) != 0 { + t.Errorf("%s: something was written", branch) + } + } +} + +func TestRulesThatCannotBeReadRefuseTheCommit(t *testing.T) { + f := newFake(t) + f.rulesBroken = true + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/2", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "protection rules") || len(f.writes) != 0 { + t.Fatalf("%v, %d writes", err, len(f.writes)) + } +} + +func TestAPatchIsAppliedByTheForgeOnANewBranch(t *testing.T) { + f := newFake(t) + patch := "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n" + got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/3", "patch": patch})) + if err != nil { + t.Fatal(err) + } + if len(f.patches) != 1 || f.patches[0]["content"] != patch || f.patches[0]["branch"] != "main" || f.patches[0]["new_branch"] != "fix/3" { + t.Fatalf("%v", f.patches) + } + if got.Commit != "p1" || len(f.writes) != 0 { + t.Fatalf("%+v", got) + } +} + +func TestACommitTheBranchDoesNotShowIsAFailure(t *testing.T) { + f := newFake(t) + f.staleRead = true + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/1-old", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "branch is at c1") { + t.Fatalf("%v", err) + } +} + +func TestDeletingAFileThatIsNotThereIsRefused(t *testing.T) { + f := newFake(t) + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/4", + "files": filesArg(map[string]any{"path": "gone.md", "delete": true})})) + if err == nil || !strings.Contains(err.Error(), "no such file") || len(f.writes) != 0 { + t.Fatalf("%v", err) + } +} + +func TestAMissingBaseIsRefused(t *testing.T) { + f := newFake(t) + _, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/5", "base": "nope", + "files": filesArg(map[string]any{"path": "x", "content": "x"})})) + if err == nil || !strings.Contains(err.Error(), "base nope is not a branch") { + t.Fatalf("%v", err) + } +} + +func TestACommitAskThatCannotBeReadIsRefused(t *testing.T) { + one := filesArg(map[string]any{"path": "x", "content": "x"}) + for name, args := range map[string]map[string]any{ + "no branch": {"files": one}, + "no message": {"branch": "b", "message": " ", "files": one}, + "neither": {"branch": "b"}, + "both": {"branch": "b", "files": one, "patch": "diff"}, + "dot path": {"branch": "b", "files": filesArg(map[string]any{"path": "a/../b", "content": "x"})}, + "absolute path": {"branch": "b", "files": filesArg(map[string]any{"path": "/etc/x", "content": "x"})}, + "twice": {"branch": "b", "files": filesArg(map[string]any{"path": "x", "content": "x"}, map[string]any{"path": "x", "delete": true})}, + "two contents": {"branch": "b", "files": filesArg(map[string]any{"path": "x", "content": "x", "delete": true})}, + "no content": {"branch": "b", "files": filesArg(map[string]any{"path": "x"})}, + "bad base64": {"branch": "b", "files": filesArg(map[string]any{"path": "x", "content_base64": "!!"})}, + "files not list": {"branch": "b", "files": "x"}, + "path with query": {"branch": "b", "files": filesArg(map[string]any{"path": "a?b", "content": "x"})}, + } { + full := map[string]any{"owner": "acme", "repo": "app", "message": "m"} + for k, v := range args { + full[k] = v + } + if _, err := readCommitAsk(full); err == nil { + t.Errorf("%s: taken", name) + } + } + f := newFake(t) + _, err := commit(context.Background(), f.serve(), CommitAsk{Owner: "acme", Repo: "app", Branch: "../x", Message: "m", Patch: "d"}) + if err == nil { + t.Error("a branch with a dot segment was taken") + } +} + +func TestRuleCoversIsTheGuardsRule(t *testing.T) { + for _, c := range []struct { + rule, branch string + want bool + }{ + {"main", "main", true}, {"main", "mainx", false}, {"release/*", "release/1.0", true}, + {"release/*", "release/a/b", false}, {"release/**", "release/a/b", true}, {"v?", "v1", true}, + {"a.b", "axb", false}, + } { + if got := ruleCovers(c.rule, c.branch); got != c.want { + t.Errorf("%s covers %s: %v", c.rule, c.branch, got) + } + } +} + +func TestHistoryAndCompareAreSummarised(t *testing.T) { + f := newFake(t) + mk := func(sha, msg string, files ...string) map[string]any { + var fs []map[string]any + for _, x := range files { + fs = append(fs, map[string]any{"filename": x, "status": "modified"}) + } + return map[string]any{"sha": sha, "commit": map[string]any{"message": msg, + "author": map[string]any{"name": "jochen", "date": "2026-10-12T10:00:00Z"}}, "files": fs} + } + f.commits = []map[string]any{mk("a1", "First line\n\nbody", "x.md", "y.md"), mk("a2", "Second", "y.md")} + forge := f.serve() + h, err := history(context.Background(), forge, "acme", "app", "fix/1", "docs", 2, 1) + if err != nil { + t.Fatal(err) + } + if len(h.Commits) != 2 || h.Commits[0].Subject != "First line" || h.Commits[0].Author != "jochen" || !h.More { + t.Fatalf("%+v", h) + } + q := f.writes[0]["query"].(string) + for _, want := range []string{"sha=fix%2F1", "path=docs", "limit=2", "files=false"} { + if !strings.Contains(q, want) { + t.Errorf("query %s lacks %s", q, want) + } + } + c, err := compare(context.Background(), forge, "acme", "app", "main", "fix/1") + if err != nil { + t.Fatal(err) + } + if c.Total != 2 || len(c.Files) != 2 || c.Files[0].Path != "x.md" || c.Commits[1].Subject != "Second" { + t.Fatalf("%+v", c) + } + if _, err := compare(context.Background(), forge, "acme", "app", "a...b", "c"); err == nil { + t.Error("a ref holding ... was compared") + } +} + +func TestReviewsSayWhichApproveTheHeadAndCarryTheirLineComments(t *testing.T) { + f := newFake(t) + f.pullHead = "h2" + f.reviews = []map[string]any{ + {"id": 1, "user": map[string]any{"login": "reviewer"}, "state": "APPROVED", "commit_id": "h1", "comments_count": 0}, + {"id": 2, "user": map[string]any{"login": "reviewer"}, "state": "REQUEST_CHANGES", "commit_id": "h2", "comments_count": 1, "body": "see line"}, + {"id": 3, "user": map[string]any{"login": "op"}, "state": "APPROVED", "commit_id": "h2", "comments_count": 0}, + } + f.comments = map[string][]map[string]any{"2": {{"path": "a.go", "position": 12, "user": map[string]any{"login": "reviewer"}, + "body": "off by one", "resolver": map[string]any{"login": "op"}}}} + r, err := reviews(context.Background(), f.serve(), "acme", "app", 7) + if err != nil { + t.Fatal(err) + } + if len(r.Reviews) != 3 || r.Reviews[0].AtHead || !r.Reviews[1].AtHead || len(r.Reviews[1].Comments) != 1 { + t.Fatalf("%+v", r) + } + lc := r.Reviews[1].Comments[0] + if lc.Path != "a.go" || lc.Line != 12 || lc.Resolved != "op" { + t.Fatalf("%+v", lc) + } + if !strings.Contains(r.Said[0], "1 approve the head") { + t.Fatalf("%v", r.Said) + } +} + +func TestADirectoryIsListed(t *testing.T) { + f := newFake(t) + f.dir = []map[string]any{{"name": "a.md", "type": "file", "size": 3}, {"name": "sub", "type": "dir", "size": 0}} + l, err := listDir(context.Background(), f.serve(), "acme", "app", "/docs/", "main") + if err != nil { + t.Fatal(err) + } + if l.Path != "/docs" || len(l.Entries) != 2 || l.Entries[0].Size != 3 || l.Entries[1].Type != "dir" { + t.Fatalf("%+v", l) + } +} + +func archive(t *testing.T, files map[string]string) []byte { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + _ = tw.WriteHeader(&tar.Header{Name: "app/", Typeflag: tar.TypeDir, Mode: 0o755}) + for name, content := range files { + if err := tw.WriteHeader(&tar.Header{Name: "app/" + name, Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(content)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +func TestCodeSearchReadsTheArchiveOnce(t *testing.T) { + f := newFake(t) + f.archive = archive(t, map[string]string{ + "docs/a.md": "one\nthe Forge\nthree forge\n", + "src/b.go": "package b // forge\n", + "bin/x": "forge\x00binary", + "docs/c.txt": "nothing\n", + }) + forge := f.serve() + found, err := searchCode(context.Background(), forge, SearchAsk{Owner: "acme", Repo: "app", Query: "forge", Max: 100}) + if err != nil { + t.Fatal(err) + } + if len(found.Matches) != 2 || found.Ref != "main" || found.Files != 3 { + t.Fatalf("%+v", found) + } + found, _ = searchCode(context.Background(), forge, SearchAsk{Owner: "acme", Repo: "app", Query: "forge", IgnoreCase: true, Path: "docs", Max: 100}) + if len(found.Matches) != 2 || found.Matches[0].Path != "docs/a.md" || found.Matches[0].Line != 2 { + t.Fatalf("%+v", found) + } + found, _ = searchCode(context.Background(), forge, SearchAsk{Owner: "acme", Repo: "app", Query: "^(one|three)", Regex: true, Max: 1}) + if len(found.Matches) != 1 || !found.Truncated { + t.Fatalf("%+v", found) + } + if _, err := searchCode(context.Background(), forge, SearchAsk{Owner: "acme", Repo: "app", Query: "(", Regex: true, Max: 1}); err == nil { + t.Error("a broken expression was searched") + } +} + +func TestAWrongPasswordFailsWithoutSayingIt(t *testing.T) { + f := newFake(t) + forge := f.serve() + f.password = "rotated" + _, err := listDir(context.Background(), forge, "acme", "app", "", "") + if err == nil || !strings.Contains(err.Error(), "401") || strings.Contains(err.Error(), "s3cret") { + t.Fatalf("%v", err) + } +} + +func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { + tools := Tools(func() (*Forge, error) { return nil, fmt.Errorf("unset") }) + var served []string + for _, tl := range tools { + served = append(served, tl.Name) + suffix := "(r)" + if tl.Name == "gitea_commit" { + suffix = "(a)" + } + if !strings.HasSuffix(tl.Description, suffix) { + t.Errorf("%s does not end with %s", tl.Name, suffix) + } + if _, err := tl.Run(map[string]any{}); err == nil { + t.Errorf("%s ran without a forge", tl.Name) + } + } + if strings.Join(served, ",") != strings.Join(ToolNames, ",") { + t.Fatalf("served %v, named %v", served, ToolNames) + } + readme, err := os.ReadFile("../../README.md") + if err != nil { + t.Fatal(err) + } + for _, n := range ToolNames { + if !strings.Contains(string(readme), "`"+n+"`") { + t.Errorf("the README does not name %s", n) + } + } + // No name the TypeScript bundle serves: two bundles serving one name is one tool answered twice. + ts, err := os.ReadFile("../../tools/index.ts") + if err != nil { + t.Fatal(err) + } + for _, n := range ToolNames { + if strings.Contains(string(ts), `name: "`+n+`"`) { + t.Errorf("the TypeScript bundle serves %s too", n) + } + } + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Tools []string `json:"tools"` + Build struct { + Artifacts []struct { + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if len(m.Tools) != 0 { + t.Error("the manifest lists tools: its claims would offer them as both seats' verbs") + } + found := false + for _, a := range m.Build.Artifacts { + if a.From == "cmd/forge" { + found = true + if len(a.Loads) != 1 || a.Loads[0] != a.Binary { + t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary) + } + for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + if a.Env[k] == "" { + t.Errorf("the bundle is not given %s", k) + } + } + } + } + if !found { + t.Error("the manifest builds no cmd/forge bundle") + } +} diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go new file mode 100644 index 00000000..7cfa04f9 --- /dev/null +++ b/modules/gitea/cmd/forge/main.go @@ -0,0 +1,236 @@ +// forge: the forge module's Go bundle for an agent's commit and the reads around it (novox/hq issue 375). +// A process the node's runtime launches and speaks MCP over stdio to, beside the module's TypeScript bundle +// and its npm-registry bundle. gitea_commit puts one commit on a branch of its own — never the default branch, +// never one a protection rule names (novox/hq ADR 0266) — and the reads answer a commit history, a compare, a +// pull request's reviews with their line comments, a directory's entries and a code search, summarised. +// stdout is the protocol; it says what it says on stderr, and never the forge's credential. +package main + +import ( + "context" + "fmt" + "math" + "os" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry +// bundle for why); the README names these, and a test holds the three together. +var ToolNames = []string{"gitea_commit", "gitea_commits", "gitea_compare", "gitea_pull_reviews", "gitea_list_dir", "gitea_search_code"} + +func main() { + if err := stdio.Serve("", Tools(ForgeFromEnv)); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// Tools are the bundle's tools over the forge the function gives; asked per call, so a manifest that does not +// set the environment is a refusal naming what is missing, not a bundle that will not start. +func Tools(forge func() (*Forge, error)) []stdio.Tool { + str := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + owner := str("the repository's owner, e.g. novox") + repo := str("the repository's name, e.g. hq") + run := func(do func(ctx context.Context, f *Forge, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + f, err := forge() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + return do(ctx, f, args) + } + } + return []stdio.Tool{ + { + Name: "gitea_commit", + Description: "Put one commit on a branch of its own in a forge repository, creating the branch from base " + + "when it does not exist. Refused on the repository's default branch and on any branch a protection " + + "rule names (novox/hq ADR 0266): a change reaches those only by a pull request whose head an " + + "independent review approved and whose required checks passed (gitea_merge_pull_request). Carries " + + "either files — each {path, content (text) | content_base64 | delete: true}, created or updated as " + + "the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " + + "Answers the commit it made and its parent, read back from the branch. Replaces git commit and git " + + "push of a feature branch. (a)", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "owner": owner, + "repo": repo, + "branch": str("the branch the commit goes on, e.g. fix/375-a-gitea-commit-tool; never the default branch or a protected one"), + "base": str("for a branch that does not exist yet: the branch it starts from (default the repository's default branch)"), + "message": str("the commit's message: concise, in the imperative, about why"), + "files": map[string]any{ + "type": "array", + "description": "the files the commit changes; give this or patch", + "items": map[string]any{ + "type": "object", + "properties": map[string]any{ + "path": str("the file's path from the repository's root"), + "content": str("its whole new content, as UTF-8 text"), + "content_base64": str("its whole new content, base64-encoded, for a file that is not text"), + "delete": map[string]any{"type": "boolean", "description": "true to delete the file"}, + }, + "required": []string{"path"}, + }, + }, + "patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this or files"), + }, + "required": []string{"owner", "repo", "branch", "message"}, + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + a, err := readCommitAsk(args) + if err != nil { + return nil, err + } + return commit(ctx, f, a) + }), + }, + { + Name: "gitea_commits", + Description: "A repository's commit history at a branch, tag or commit (default the default branch), " + + "newest first, optionally only the commits touching a path: each commit's sha, author, date and " + + "subject line. Replaces git log. (r)", + Input: map[string]any{ + "owner": owner, "repo": repo, + "ref": str("a branch, tag or commit (default the default branch)"), + "path": str("only commits touching this file or directory"), + "limit": map[string]any{"type": "integer", "description": "commits per page, 1 to 100 (default 20)"}, + "page": map[string]any{"type": "integer", "description": "the page, from 1 (default 1)"}, + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + limit, err := count(args, "limit", 20, 1, 100) + if err != nil { + return nil, err + } + page, err := count(args, "page", 1, 1, 10000) + if err != nil { + return nil, err + } + return history(ctx, f, text(args, "owner"), text(args, "repo"), text(args, "ref"), strings.Trim(text(args, "path"), "/"), limit, page) + }), + }, + { + Name: "gitea_compare", + Description: "What head holds that base does not, in a forge repository: its commits (sha, author, date, " + + "subject) and the files they touch with how. Replaces git log base..head and git diff --stat. (r)", + Input: map[string]any{ + "owner": owner, "repo": repo, + "base": str("the branch, tag or commit compared from, e.g. main"), + "head": str("the branch, tag or commit compared, e.g. fix/375-a-gitea-commit-tool"), + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + return compare(ctx, f, text(args, "owner"), text(args, "repo"), text(args, "base"), text(args, "head")) + }), + }, + { + Name: "gitea_pull_reviews", + Description: "A pull request's reviews, oldest first: who, its state (APPROVED, REQUEST_CHANGES, COMMENT, " + + "PENDING), the commit it reviewed and whether that is the pull request's head now, stale or dismissed, " + + "its body, and every line comment it left (file, line, body, who resolved it). (r)", + Input: map[string]any{ + "owner": owner, "repo": repo, + "number": map[string]any{"type": "integer", "description": "the pull request's number"}, + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + n, err := count(args, "number", 0, 1, math.MaxInt32) + if err != nil { + return nil, err + } + if n == 0 { + return nil, refuse("number is required: the pull request's number") + } + return reviews(ctx, f, text(args, "owner"), text(args, "repo"), n) + }), + }, + { + Name: "gitea_list_dir", + Description: "A directory's entries in a forge repository at a branch, tag or commit (default the default " + + "branch): each entry's name, its type (file, dir, symlink, submodule) and a file's size. Replaces ls " + + "and git ls-tree. (r)", + Input: map[string]any{ + "owner": owner, "repo": repo, + "path": str("the directory from the repository's root (default the root)"), + "ref": str("a branch, tag or commit (default the default branch)"), + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + return listDir(ctx, f, text(args, "owner"), text(args, "repo"), text(args, "path"), text(args, "ref")) + }), + }, + { + Name: "gitea_search_code", + Description: "Search a forge repository's text files at a branch, tag or commit (default the default " + + "branch) for a literal string, or a regular expression with regex: true, optionally under a path: " + + "each matching line's file, number and text. The forge has no code search of its own, so the " + + "repository's archive is read once and searched; files over 4 MiB are named, not searched. Replaces " + + "git grep. (r)", + Input: map[string]any{ + "owner": owner, "repo": repo, + "query": str("what to find: a literal string, or a Go regular expression with regex: true"), + "regex": map[string]any{"type": "boolean", "description": "true to read query as a regular expression"}, + "ignore_case": map[string]any{"type": "boolean", "description": "true to match regardless of case"}, + "path": str("only files under this directory, or this one file"), + "ref": str("a branch, tag or commit (default the default branch)"), + "max": map[string]any{"type": "integer", "description": "the most matches answered, 1 to 1000 (default 100)"}, + }, + Run: run(func(ctx context.Context, f *Forge, args map[string]any) (any, error) { + max, err := count(args, "max", 100, 1, 1000) + if err != nil { + return nil, err + } + regex, err := flag(args, "regex") + if err != nil { + return nil, err + } + ignore, err := flag(args, "ignore_case") + if err != nil { + return nil, err + } + return searchCode(ctx, f, SearchAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Ref: text(args, "ref"), + Query: raw(args, "query"), Path: text(args, "path"), Regex: regex, IgnoreCase: ignore, Max: max}) + }), + }, + } +} + +// text is an argument trimmed; raw is it as given. +func text(args map[string]any, key string) string { return strings.TrimSpace(raw(args, key)) } + +func raw(args map[string]any, key string) string { + s, _ := args[key].(string) + return s +} + +func count(args map[string]any, key string, fallback, least, most int) (int, error) { + v, given := args[key] + if !given || v == nil { + return fallback, nil + } + x, ok := v.(float64) + if !ok || x != math.Trunc(x) { + return 0, refuse("%s must be a whole number", key) + } + if x < float64(least) || x > float64(most) { + return 0, refuse("%s must be from %d to %d", key, least, most) + } + return int(x), nil +} + +func flag(args map[string]any, key string) (bool, error) { + v, given := args[key] + if !given || v == nil { + return false, nil + } + switch b := v.(type) { + case bool: + return b, nil + case string: + if b == "true" || b == "false" { + return b == "true", nil + } + } + return false, refuse("%s is true or false, not %v", key, v) +} diff --git a/modules/gitea/cmd/forge/reads.go b/modules/gitea/cmd/forge/reads.go new file mode 100644 index 00000000..464a4b85 --- /dev/null +++ b/modules/gitea/cmd/forge/reads.go @@ -0,0 +1,334 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +// The forge's reads an agent otherwise made through gitea_api, answered summarised rather than as the forge's +// JSON (novox/hq issue 375): a commit's history, a compare, a pull request's reviews with their line comments, +// a directory's entries. + +// CommitLine is one commit, summarised. +type CommitLine struct { + SHA string `json:"sha"` + Author string `json:"author"` + Date string `json:"date"` + Subject string `json:"subject"` + Files int `json:"files,omitempty"` +} + +type forgeCommit struct { + SHA string `json:"sha"` + Commit struct { + Message string `json:"message"` + Author struct { + Name string `json:"name"` + Email string `json:"email"` + Date string `json:"date"` + } `json:"author"` + } `json:"commit"` + Files []struct { + Filename string `json:"filename"` + Status string `json:"status"` + } `json:"files"` +} + +func (c forgeCommit) line() CommitLine { + subject, _, _ := strings.Cut(strings.TrimSpace(c.Commit.Message), "\n") + return CommitLine{SHA: c.SHA, Author: c.Commit.Author.Name, Date: c.Commit.Author.Date, Subject: subject, Files: len(c.Files)} +} + +// History is a ref's commits, newest first. +type History struct { + Repository string `json:"repository"` + Ref string `json:"ref,omitempty"` + Path string `json:"path,omitempty"` + Page int `json:"page"` + Commits []CommitLine `json:"commits"` + More bool `json:"more"` +} + +func history(ctx context.Context, f *Forge, owner, name, ref, path string, limit, page int) (*History, error) { + repo, err := repoPath(owner, name) + if err != nil { + return nil, err + } + q := url.Values{} + if ref != "" { + q.Set("sha", ref) + } + if path != "" { + q.Set("path", path) + } + q.Set("limit", fmt.Sprint(limit)) + q.Set("page", fmt.Sprint(page)) + q.Set("stat", "false") + q.Set("verification", "false") + q.Set("files", "false") + var got []forgeCommit + if err := f.getJSON(ctx, repo+"/commits?"+q.Encode(), &got); err != nil { + return nil, err + } + h := &History{Repository: owner + "/" + name, Ref: ref, Path: path, Page: page, Commits: []CommitLine{}, More: len(got) >= limit} + for _, c := range got { + l := c.line() + l.Files = 0 + h.Commits = append(h.Commits, l) + } + return h, nil +} + +// Comparison is what head holds that base does not. +type Comparison struct { + Repository string `json:"repository"` + Base string `json:"base"` + Head string `json:"head"` + Total int `json:"total_commits"` + Commits []CommitLine `json:"commits"` + Files []FileLine `json:"files"` + Said []string `json:"said"` +} + +// FileLine is one file a compare touched, and how the last commit that touched it changed it. +type FileLine struct { + Path string `json:"path"` + Status string `json:"status"` +} + +func compare(ctx context.Context, f *Forge, owner, name, base, head string) (*Comparison, error) { + repo, err := repoPath(owner, name) + if err != nil { + return nil, err + } + b, err := segments("base", base) + if err != nil { + return nil, err + } + h, err := segments("head", head) + if err != nil { + return nil, err + } + if strings.Contains(base, "...") || strings.Contains(head, "...") { + return nil, refuse("a base or head holding ... cannot be compared: the forge splits on it. Nothing was sent") + } + var got struct { + Total int `json:"total_commits"` + Commits []forgeCommit `json:"commits"` + } + if err := f.getJSON(ctx, repo+"/compare/"+b+"..."+h, &got); err != nil { + return nil, err + } + c := &Comparison{Repository: owner + "/" + name, Base: base, Head: head, Total: got.Total, Commits: []CommitLine{}, Files: []FileLine{}} + at := map[string]int{} + for _, fc := range got.Commits { + c.Commits = append(c.Commits, fc.line()) + for _, file := range fc.Files { + if i, seen := at[file.Filename]; seen { + c.Files[i].Status = file.Status + continue + } + at[file.Filename] = len(c.Files) + c.Files = append(c.Files, FileLine{Path: file.Filename, Status: file.Status}) + } + } + c.Said = append(c.Said, fmt.Sprintf("%s holds %d commits %s does not, touching %d files", head, got.Total, base, len(c.Files))) + if got.Total > len(got.Commits) { + c.Said = append(c.Said, fmt.Sprintf("the forge listed %d of the %d commits; the files are those of the commits listed", len(got.Commits), got.Total)) + } + c.Said = append(c.Said, "a file's status is the last listed commit's change to it; for the lines, gitea_pull_request_diff or gitea_get_file") + return c, nil +} + +// Review is one review of a pull request with its line comments. +type Review struct { + ID int64 `json:"id"` + By string `json:"by"` + State string `json:"state"` + Commit string `json:"commit"` + AtHead bool `json:"at_head"` + Submitted string `json:"submitted,omitempty"` + Stale bool `json:"stale,omitempty"` + Dismissed bool `json:"dismissed,omitempty"` + Official bool `json:"official,omitempty"` + Body string `json:"body,omitempty"` + Comments []LineComment `json:"comments"` +} + +// LineComment is one comment a review left on a line. +type LineComment struct { + Path string `json:"path"` + Line int `json:"line,omitempty"` + OldLine int `json:"old_line,omitempty"` + By string `json:"by"` + Body string `json:"body"` + Commit string `json:"commit,omitempty"` + Resolved string `json:"resolved_by,omitempty"` +} + +// Reviews is a pull request's reviews, oldest first. +type Reviews struct { + Repository string `json:"repository"` + Number int `json:"number"` + Head string `json:"head"` + Reviews []Review `json:"reviews"` + Said []string `json:"said"` +} + +type forgeUser struct { + Login string `json:"login"` +} + +func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Reviews, error) { + repo, err := repoPath(owner, name) + if err != nil { + return nil, err + } + pr := fmt.Sprintf("%s/pulls/%d", repo, number) + var pull struct { + Head struct { + SHA string `json:"sha"` + } `json:"head"` + } + if err := f.getJSON(ctx, pr, &pull); err != nil { + if errors.Is(err, errNotFound) { + return nil, refuse("%s/%s has no pull request %d", owner, name, number) + } + return nil, err + } + out := &Reviews{Repository: owner + "/" + name, Number: number, Head: pull.Head.SHA, Reviews: []Review{}} + type forgeReview struct { + ID int64 `json:"id"` + User forgeUser `json:"user"` + State string `json:"state"` + Body string `json:"body"` + CommitID string `json:"commit_id"` + Submitted string `json:"submitted_at"` + Stale bool `json:"stale"` + Dismissed bool `json:"dismissed"` + Official bool `json:"official"` + Comments int `json:"comments_count"` + } + const page = 50 + var all []forgeReview + for p := 1; ; p++ { + var batch []forgeReview + if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews?limit=%d&page=%d", pr, page, p), &batch); err != nil { + return nil, err + } + all = append(all, batch...) + if len(batch) < page || p >= 20 { + break + } + } + approvedAtHead := 0 + for _, r := range all { + rv := Review{ID: r.ID, By: r.User.Login, State: r.State, Commit: r.CommitID, AtHead: r.CommitID != "" && r.CommitID == pull.Head.SHA, + Submitted: r.Submitted, Stale: r.Stale, Dismissed: r.Dismissed, Official: r.Official, Body: r.Body, Comments: []LineComment{}} + if r.Comments > 0 { + var cs []struct { + Path string `json:"path"` + Position int `json:"position"` + Original int `json:"original_position"` + User forgeUser `json:"user"` + Body string `json:"body"` + CommitID string `json:"commit_id"` + Resolver *forgeUser `json:"resolver"` + } + if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews/%d/comments", pr, r.ID), &cs); err != nil { + return nil, fmt.Errorf("review %d's line comments could not be read: %w", r.ID, err) + } + for _, c := range cs { + lc := LineComment{Path: c.Path, Line: c.Position, OldLine: c.Original, By: c.User.Login, Body: c.Body, Commit: c.CommitID} + if c.Resolver != nil { + lc.Resolved = c.Resolver.Login + } + rv.Comments = append(rv.Comments, lc) + } + } + if rv.AtHead && r.State == "APPROVED" && !r.Dismissed && !r.Stale { + approvedAtHead++ + } + out.Reviews = append(out.Reviews, rv) + } + out.Said = append(out.Said, fmt.Sprintf("%d reviews; %d approve the head %s", len(out.Reviews), approvedAtHead, short(pull.Head.SHA))) + return out, nil +} + +// Entry is one entry of a directory. +type Entry struct { + Name string `json:"name"` + Type string `json:"type"` + Size int64 `json:"size,omitempty"` +} + +// Listing is a directory's entries at a ref. +type Listing struct { + Repository string `json:"repository"` + Path string `json:"path"` + Ref string `json:"ref,omitempty"` + Entries []Entry `json:"entries"` +} + +func listDir(ctx context.Context, f *Forge, owner, name, path, ref string) (*Listing, error) { + repo, err := repoPath(owner, name) + if err != nil { + return nil, err + } + path = strings.Trim(path, "/") + at := repo + "/contents" + if path != "" { + seg, err := segments("directory path", path) + if err != nil { + return nil, err + } + at += "/" + seg + } + if ref != "" { + at += "?ref=" + queryEscape(ref) + } + body, err := f.do(ctx, "GET", at, nil, maxBody) + if err != nil { + if errors.Is(err, errNotFound) { + return nil, refuse("%s/%s has no %q at %s", owner, name, path, orDefault(ref)) + } + return nil, err + } + var got []struct { + Name string `json:"name"` + Type string `json:"type"` + Size int64 `json:"size"` + } + if err := jsonInto(body, &got); err != nil { + return nil, refuse("%s is a file, not a directory: gitea_get_file reads it", path) + } + l := &Listing{Repository: owner + "/" + name, Path: "/" + path, Ref: ref, Entries: []Entry{}} + for _, e := range got { + en := Entry{Name: e.Name, Type: e.Type} + if e.Type == "file" { + en.Size = e.Size + } + l.Entries = append(l.Entries, en) + } + return l, nil +} + +func orDefault(ref string) string { + if ref == "" { + return "the default branch" + } + return ref +} + +func jsonInto(body []byte, into any) error { return json.Unmarshal(body, into) } + +// queryEscape is a query value as the forge reads it. +func queryEscape(v string) string { return url.QueryEscape(v) } + +// deadline is how long one tool call may take: inside a module's 60-second ask. +const deadline = 50 * time.Second diff --git a/modules/gitea/cmd/forge/search.go b/modules/gitea/cmd/forge/search.go new file mode 100644 index 00000000..3931ced2 --- /dev/null +++ b/modules/gitea/cmd/forge/search.go @@ -0,0 +1,171 @@ +package main + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" +) + +// gitea_search_code: the forge's API has no code search, so the repository's archive at a ref is read once +// and searched here (novox/hq issue 375). One request, nothing kept. + +// Match is one line that matched. +type Match struct { + Path string `json:"path"` + Line int `json:"line"` + Text string `json:"text"` +} + +// Found is a search's answer. +type Found struct { + Repository string `json:"repository"` + Ref string `json:"ref"` + Query string `json:"query"` + Matches []Match `json:"matches"` + Files int `json:"files_searched"` + Skipped []string `json:"skipped,omitempty"` + Truncated bool `json:"truncated"` + Said []string `json:"said"` +} + +// SearchAsk is one gitea_search_code call. +type SearchAsk struct { + Owner, Repo, Ref, Query, Path string + Regex, IgnoreCase bool + Max int +} + +const ( + maxFileSearched = 4 << 20 // a larger file is skipped and named + maxLineShown = 300 +) + +func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { + repo, err := repoPath(a.Owner, a.Repo) + if err != nil { + return nil, err + } + if a.Query == "" { + return nil, refuse("query is required. Nothing was sent") + } + pattern := a.Query + if !a.Regex { + pattern = regexp.QuoteMeta(pattern) + } + if a.IgnoreCase { + pattern = "(?i)" + pattern + } + re, err := regexp.Compile(pattern) + if err != nil { + return nil, refuse("query is not a regular expression Go reads: %v. Nothing was sent", err) + } + ref := a.Ref + if ref == "" { + var r Repo + if err := f.getJSON(ctx, repo, &r); err != nil { + return nil, err + } + if r.Empty || r.DefaultBranch == "" { + return &Found{Repository: a.Owner + "/" + a.Repo, Query: a.Query, Matches: []Match{}, Said: []string{"the repository is empty"}}, nil + } + ref = r.DefaultBranch + } + seg, err := segments("ref", ref) + if err != nil { + return nil, err + } + prefix := strings.Trim(a.Path, "/") + if prefix != "" { + if _, err := segments("path", prefix); err != nil { + return nil, err + } + } + body, err := f.do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive) + if err != nil { + if errors.Is(err, errNotFound) { + return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref) + } + return nil, err + } + out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}} + if err := searchArchive(body, prefix, re, a.Max, out); err != nil { + return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err) + } + said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files) + if out.Truncated { + said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches)) + } + out.Said = append(out.Said, said) + if len(out.Skipped) > 0 { + out.Said = append(out.Said, fmt.Sprintf("%d files larger than %d MiB were not searched: they are under skipped", len(out.Skipped), maxFileSearched>>20)) + } + return out, nil +} + +// searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the +// lines that match, from text files under the prefix. +func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error { + zr, err := gzip.NewReader(bytes.NewReader(gz)) + if err != nil { + return err + } + tr := tar.NewReader(zr) + for { + h, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if h.Typeflag != tar.TypeReg { + continue + } + _, path, found := strings.Cut(h.Name, "/") + if !found || path == "" { + continue + } + if prefix != "" && path != prefix && !strings.HasPrefix(path, prefix+"/") { + continue + } + if h.Size > maxFileSearched { + out.Skipped = append(out.Skipped, path) + continue + } + content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1)) + if err != nil { + return err + } + head := content + if len(head) > 8000 { + head = head[:8000] + } + if bytes.IndexByte(head, 0) >= 0 { + continue // binary + } + out.Files++ + sc := bufio.NewScanner(bytes.NewReader(content)) + sc.Buffer(make([]byte, 64<<10), maxFileSearched+1) + for n := 1; sc.Scan(); n++ { + line := sc.Text() + if !re.MatchString(line) { + continue + } + if len(out.Matches) >= max { + out.Truncated = true + return nil + } + if len(line) > maxLineShown { + line = strings.ToValidUTF8(line[:maxLineShown], "") + "…" + } + out.Matches = append(out.Matches, Match{Path: path, Line: n, Text: line}) + } + } +} diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 7358ebde..6280acff 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -257,6 +257,22 @@ "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_NPM_OWNER": "novox" } + }, + { + "name": "forge", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/forge", + "binary": "forge", + "loads": [ + "forge" + ], + "env": { + "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", + "MESH_GITEA_ADMIN_USER": "mesh-admin", + "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" + } } ] },