// Package bridge turns the seat git's pull request events into the messages the forge bridge writes on FORGE. package bridge import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "maps" "path" "regexp" "slices" "strconv" "strings" "sync" git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" ) // ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never // redelivered. var ErrUnreadable = errors.New("unreadable event") // ErrNotHandled says the event is one the bridge hears but writes nothing for (yet). var ErrNotHandled = errors.New("event not handled") // ErrRetry says the event is readable but the forge could not be asked what the message needs; the event is to be // heard again later. var ErrRetry = errors.New("the forge could not say") // Watching says whether the bridge watches a branch of a repository (Watched; design 47 G §507, §532). type Watching interface { Watches(repository, branch string) bool } // Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json // (the root directory spelled ""), as the seat git's verb trees answers (issue 1234/AC4). type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error) // ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448). var ErrNotFound = errors.New("the forge has no such file") // Files asks the forge for one file's bytes at a ref (a commit), as the seat git's verb file answers; a file the forge // says is not there is ErrNotFound, any other failure any other error (issue 1448). type Files func(owner, repo, path, ref string) ([]byte, error) // CatalogRepository is the catalogue, whose merged message carries its predefined list, and PredefinedFile that list's // file at its root (novox/hq ADR 0565 §2, ADR 0348 §7). const ( CatalogRepository = "novox/mesh-catalog" PredefinedFile = "predefined.json" ) // LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465 // clause 1: go.sum for a Go module, package-lock.json for a Node one). var LockFiles = []string{"go.sum", "package-lock.json"} // Message is a message for FORGE: the event name (forge...) and its JSON body. type Message struct { Event string Data []byte // Said is what the bridge did that the log says, beside writing the message; empty for none. Said string } // The subjects of the seat git's events the bridge writes pull messages for (novox/hq design 47 *The seat `git`*): the // seat's holder says them on mesh.seat.git.event., whichever module holds it; the bridge names none. const ( UpdatedSubject = git.SubjectPullUpdated ClosedSubject = git.SubjectPullClosed MergedSubject = git.SubjectPullMerged // DeletedSubject is the seat's forge.delete: a branch or a tag deleted (repository, ref, ref-type). Only the seat's // holder may publish on the seat's subjects (novox/hq ADR 0363 §2), so no other module can end a branch's tagged // versions. DeletedSubject = git.SubjectForgeDelete ) // ForgeBranchDeleted is the kind of a branch-deleted message on FORGE (novox/hq design 51 *The forge bridge says a // branch's end*): the builder removes the branch's tagged versions on it, as on a pull request's merged or closed. // Every existing consumer of FORGE ignores a kind it does not know. const ForgeBranchDeleted = "branch-deleted" // BranchDeleted is the body of a branch-deleted message. type BranchDeleted struct { V int `json:"v"` Kind string `json:"kind"` Repository string `json:"repository"` Branch string `json:"branch"` // Head is the commit the branch named when it was deleted, when the forge said it. Head string `json:"head,omitempty"` } // branchDeleted turns forge.delete into a branch-deleted message; a deleted tag is not handled. func branchDeleted(subject string, data []byte, watched Watching) (Message, error) { var e git.ForgeDeleteEvent if err := json.Unmarshal(data, &e); err != nil { return Message{}, unreadable(subject, "the body is not json: %v", err) } d := struct{ Owner, Repo, Ref, RefType string }{Ref: e.Ref, RefType: e.RefType} d.Owner, d.Repo, _ = strings.Cut(e.Repository, "/") if d.RefType != "branch" { return Message{}, fmt.Errorf("%w: %s of a %s (%s)", ErrNotHandled, subject, d.RefType, d.Ref) } branch := strings.TrimPrefix(d.Ref, "refs/heads/") switch { case d.Owner == "" || d.Repo == "" || strings.ContainsAny(d.Owner+d.Repo, "./ *>"): return Message{}, unreadable(subject, "no owner or repo a forge event can name (%q/%q)", d.Owner, d.Repo) case branch == "": return Message{}, unreadable(subject, "no branch") case watched.Watches(d.Owner+"/"+d.Repo, branch): return Message{}, fmt.Errorf("%w: %s of the watched branch %s", ErrNotHandled, subject, branch) } body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch}) if err != nil { return Message{}, unreadable(subject, "%v", err) } return Message{Event: "forge." + d.Owner + "." + d.Repo + "." + ForgeBranchDeleted, Data: body, Said: fmt.Sprintf("branch %s of %s/%s deleted: its tagged versions end", branch, d.Owner, d.Repo)}, nil } var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`) // pull is the part of the seat git's pull.updated, pull.merged and pull.closed the bridge reads: Head is the branch and // HeadSHA the commit, Owner and Repo the repository's two halves. type pull struct { Owner, Repo string Number int Head, HeadSHA string ModuleDirs []string ModuleDirsSaid bool Base string MergeSHA string } // pullOf reads one of the seat's pull events. The directories are the seat's word only when its change says them: an // absent `module-dirs` is "could not be read", never "none". func pullOf(data []byte) (pull, error) { var e struct { Repository string `json:"repository"` Number int64 `json:"number"` Branch string `json:"branch"` Head string `json:"head"` Base string `json:"base"` MergeCommit string `json:"merge-commit"` Change json.RawMessage `json:"change"` } if err := json.Unmarshal(data, &e); err != nil { return pull{}, err } p := pull{Number: int(e.Number), Head: e.Branch, HeadSHA: e.Head, Base: e.Base, MergeSHA: e.MergeCommit} p.Owner, p.Repo, _ = strings.Cut(e.Repository, "/") if len(e.Change) > 0 && string(e.Change) != "null" { var c map[string]json.RawMessage if err := json.Unmarshal(e.Change, &c); err != nil { return pull{}, fmt.Errorf("its change: %w", err) } if raw, said := c["module-dirs"]; said { if err := json.Unmarshal(raw, &p.ModuleDirs); err != nil { return pull{}, fmt.Errorf("its change's module-dirs: %w", err) } p.ModuleDirsSaid = true } } return p, nil } func unreadable(subject, why string, args ...any) error { return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...)) } // Convert turns one pull request event of the seat git, by its bus subject and body, into a message. An event it cannot // read is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A pull // request into a branch the bridge does not watch is no candidate: written closed. A merge into a watched branch writes // nothing here: its merged message is the move's, heard on branch-moved (MoveMessage; design 47 G §509, §714); a merge // into another branch is a closed pull message, as the pull request left its group. A forge that cannot say is ErrRetry. func Convert(subject string, data []byte, files Files, watched Watching) (Message, error) { var action string switch subject { case UpdatedSubject: action = forgemsg.PullUpdated case ClosedSubject, MergedSubject: action = forgemsg.PullClosed case DeletedSubject: return branchDeleted(subject, data, watched) default: return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject) } p, err := pullOf(data) if err != nil { return Message{}, unreadable(subject, "the body is not the seat's json: %v", err) } switch { case p.Owner == "" || p.Repo == "" || strings.Contains(p.Repo, "/"): return Message{}, unreadable(subject, "no owner or repo") case p.Number <= 0: return Message{}, unreadable(subject, "no pull request number") case !sha40.MatchString(p.HeadSHA): return Message{}, unreadable(subject, "the head %q is not 40 hex digits", p.HeadSHA) case p.Head == "": return Message{}, unreadable(subject, "no branch") } into := p.Base != "" && watched.Watches(p.Owner+"/"+p.Repo, p.Base) if subject == MergedSubject && into { return Message{}, fmt.Errorf("%w: %s of %s/%s#%d into the watched branch %s: its merged message is written from the branch's move", ErrNotHandled, subject, p.Owner, p.Repo, p.Number, p.Base) } said := "" if action == forgemsg.PullUpdated && p.Base != "" && !into { // A pull request into a branch no registration watches is no candidate, as it is none at the start (OpenPulls) and at // its merge. It is written closed, so a candidate an older head made is taken out, and said (issue 1422). action = forgemsg.PullClosed said = fmt.Sprintf("%s/%s#%d at %s is into %s, which is not watched: written closed, so it is no candidate", p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base) } f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} // The directories are the forge's word only when it said it could read them (module_dirs_said). if action == forgemsg.PullUpdated && p.ModuleDirsSaid { f.Dirs, f.DirsKnown = p.ModuleDirs, true var err error if f.Manifests, f.Locks, f.Needs, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) } } event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := forgemsg.EncodeForge(f) if err != nil { return Message{}, unreadable(subject, "%v", err) } return Message{Event: event, Data: body, Said: said}, nil } // MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many // modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event // (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound // (issue 1458). const MaxReadsInFlight = 8 // read is one file asked for: the path in the directory dir, and whether it is the module's manifest. type read struct { dir, path string manifest bool asked bool raw []byte err error } // inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it // (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge // does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other // failure is an error (issue 1448). // // The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the // order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After // a failure no further file is asked for, and inputsOf returns only when every call it started has returned. func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) { manifests, locks, _, _, err = inputsRead(owner, repo, ref, dirs, files) return manifests, locks, err } // inputsRead is inputsOf, and also answers, on a failure too, the directories whose manifest and lock files were all // read, with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest // (issue 1126). It reads no mesh package a go.sum names: the indexer alone computes what a fingerprint takes (novox/hq // ADR 0593 §2), so the message's Packages stay empty. Of a directory whose manifest names templates it reads each // template too, in a second round, and answers the module's needs whole (catalogue.ModuleNeeds; novox/hq ADR 0632 §2): // a template's text can need a reading its manifest does not. A template the forge does not have is no need (the judge // refuses the manifest by name); any other failure to read one is the directory's, as a lock file's is. func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, needs map[string][]string, whole []string, err error) { at := func(dir, name string) string { if dir == "" { return name } return dir + "/" + name } var reads []*read for _, dir := range dirs { reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true}) for _, name := range LockFiles { reads = append(reads, &read{dir: dir, path: at(dir, name)}) } } fetch(owner, repo, ref, reads, files) manifests = map[string]string{} locks = map[string]map[string]string{} broken := map[string]bool{} // a directory with a file not read (failed, or never asked after a failure) failed := func(r *read) bool { switch { case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)): broken[r.dir] = true if err == nil { err = fmt.Errorf("%s: %w", r.path, r.err) } case r.err == nil && r.raw == nil && !r.asked: broken[r.dir] = true default: return false } return true } for _, r := range reads { switch { case failed(r): case r.err != nil: // a lock file the forge does not have case r.manifest: manifests[r.dir] = string(r.raw) default: sum := sha256.Sum256(r.raw) if locks[r.dir] == nil { locks[r.dir] = map[string]string{} } locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:]) } } var templates []*read if err == nil { for _, dir := range dirs { for _, t := range templatePaths(manifests[dir]) { templates = append(templates, &read{dir: dir, path: at(dir, t)}) } } fetch(owner, repo, ref, templates, files) } texts := map[string]map[string]string{} for _, r := range templates { if failed(r) || r.err != nil { continue } if texts[r.dir] == nil { texts[r.dir] = map[string]string{} } texts[r.dir][strings.TrimPrefix(r.path, at(r.dir, ""))] = string(r.raw) } needs = map[string][]string{} for _, dir := range dirs { if raw, ok := manifests[dir]; ok && !broken[dir] && len(templatePaths(raw)) > 0 { needs[dir], _ = catalogue.ModuleNeeds([]byte(raw), texts[dir]) } } for _, dir := range dirs { if !broken[dir] { whole = append(whole, dir) } } if err != nil { for dir := range broken { delete(manifests, dir) delete(locks, dir) delete(needs, dir) } return manifests, locks, needs, whole, err } if len(manifests) == 0 { manifests = nil } if len(locks) == 0 { locks = nil } if len(needs) == 0 { needs = nil } return manifests, locks, needs, whole, nil } // templatePaths is the templates a manifest names (catalogue.TemplatePaths), none for one that does not decode. func templatePaths(raw string) []string { if raw == "" { return nil } m, err := catalogue.Decode([]byte(raw)) if err != nil { return nil } return catalogue.TemplatePaths(m) } // fetch asks the forge for each of reads at ref, at most MaxReadsInFlight at once; after a failure (a manifest, or a // file the forge has and could not give) no further file is asked for, and it returns once every call it started has. func fetch(owner, repo, ref string, reads []*read, files Files) { var ( wg sync.WaitGroup mu sync.Mutex failed bool slots = make(chan struct{}, MaxReadsInFlight) ) for _, r := range reads { slots <- struct{}{} mu.Lock() stop := failed mu.Unlock() if stop { <-slots break } r.asked = true wg.Add(1) go func(r *read) { defer func() { <-slots; wg.Done() }() r.raw, r.err = files(owner, repo, r.path, ref) if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) { mu.Lock() failed = true mu.Unlock() } }(r) } wg.Wait() } // landed is the message of a commit that landed on a watched branch, merged or pushed (f says which and what names it): // the trees of the commit, asked once, and the take-in inputs of every module directory at it (design 47 G §509, A-236). func landed(what string, f forgemsg.Forge, commit string, trees Trees, files Files, kept Kept) (Message, error) { if !sha40.MatchString(commit) { return Message{}, unreadable(what, "the commit %q is not 40 hex digits", commit) } repository := f.Repository owner, repo, _ := strings.Cut(repository, "/") root, dirs, err := trees(owner, repo, commit) if err != nil { return Message{}, fmt.Errorf("%w: %s: the trees of %s at %s: %v", ErrRetry, what, repository, commit, err) } names := make([]string, 0, len(dirs)) for dir := range dirs { names = append(names, dir) } slices.Sort(names) manifests, locks, needs, toRead := keptInputs(names, dirs, kept) read, readLocks, readNeeds, whole, err := inputsRead(owner, repo, commit, toRead, files) if err != nil { keep(whole, dirs, read, readLocks, readNeeds, kept) // what was read whole is not asked again at the next try return Message{}, fmt.Errorf("%w: %s: the manifests of %s at %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, what, repository, commit, len(toRead), len(names), err) } keep(toRead, dirs, read, readLocks, readNeeds, kept) maps.Copy(manifests, read) maps.Copy(locks, readLocks) maps.Copy(needs, readNeeds) if len(manifests) == 0 { manifests = nil } if len(locks) == 0 { locks = nil } if len(needs) == 0 { needs = nil } event, err := forgemsg.ForgeEvent(repository, f.Kind) if err != nil { return Message{}, unreadable(what, "%v", err) } f.Root, f.Trees, f.Manifests, f.Locks, f.Needs = root, dirs, manifests, locks, needs if repository == CatalogRepository { // The catalogue's predefined list at the commit (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. raw, err := files(owner, repo, PredefinedFile, commit) switch { case errors.Is(err, ErrNotFound): f.PredefinedRead = true case err != nil: return Message{}, fmt.Errorf("%w: %s: %s of %s at %s: %v", ErrRetry, what, PredefinedFile, repository, commit, err) default: f.Predefined = json.RawMessage(raw) } } body, err := forgemsg.EncodeForge(f) if err != nil { return Message{}, unreadable(what, "%v", err) } msg := Message{Event: event, Data: body} if len(dirs) == 0 { msg.Said = fmt.Sprintf("%s at %s holds no module.json: a %s message with no module directories", repository, commit, f.Kind) } return msg, nil } // openPulls reads the seat git's answer of open-pulls: an answer saying an error is the error, never none open. func openPulls(answer []byte) ([]pull, error) { var a git.OpenPullsAnswer if err := json.Unmarshal(answer, &a); err != nil { return nil, fmt.Errorf("the seat git's answer of open-pulls is not its contract's: %v", err) } if a.Error != "" { return nil, fmt.Errorf("the seat git refused open-pulls: %s", a.Error) } var said map[string]json.RawMessage if _ = json.Unmarshal(answer, &said); said["pulls"] == nil || string(said["pulls"]) == "null" { // None open is `pulls: []`; an answer naming no list would drop every candidate on a holder's fault. return nil, fmt.Errorf("open-pulls: %w", errNoAnswer) } out := make([]pull, 0, len(a.Pulls)) for _, o := range a.Pulls { p := pull{Number: int(o.Pull.Number), Head: o.Pull.Branch, HeadSHA: o.Pull.Head, Base: o.Pull.Base} p.Owner, p.Repo, _ = strings.Cut(o.Pull.Repository, "/") out = append(out, p) } return out, nil } // OpenPulls turns the seat git's answer of open-pulls into one pull message per open pull request into // a watched branch, action updated and its directories not known (the forge's answer does not say them). A pull request into // another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not // the contract's, or that says an error, is an error. func OpenPulls(answer []byte, watched Watching) (msgs []Message, skipped []string, err error) { pulls, err := openPulls(answer) if err != nil { return nil, nil, err } for _, p := range pulls { if !watched.Watches(p.Owner+"/"+p.Repo, p.Base) { continue } name := fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) if p.Owner == "" || p.Repo == "" || p.Number <= 0 || p.Head == "" || !sha40.MatchString(p.HeadSHA) { skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head") continue } f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: forgemsg.PullUpdated} event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull) var body []byte if err == nil { body, err = forgemsg.EncodeForge(f) } if err != nil { skipped = append(skipped, name+": "+err.Error()) continue } msgs = append(msgs, Message{Event: event, Data: body}) } return msgs, skipped, nil } // Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads, // novox/hq ADR 0443 §5): the key `..`, and the head to keep under it, or "" to take the key away // (a closed pull request, a merged one). ok is false for a message that keeps no head. func Track(f forgemsg.Forge) (key, head string, ok bool) { key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number) switch { case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed: return key, f.Head, true case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged: return key, "", true } return "", "", false } // OpenMessage turns the seat git's answer of open-pulls into the open message: every open pull request into a watched branch, by // repository and number, whether its head reads or not and whether FORGE holds it or not, in repository and number // order. A pull request into another branch is not named. An answer that names none is a message too: nothing is open. func OpenMessage(answer []byte, watched Watching) (Message, error) { pulls, err := openPulls(answer) if err != nil { return Message{}, err } open := []forgemsg.OpenPull{} for _, p := range pulls { if !watched.Watches(p.Owner+"/"+p.Repo, p.Base) || p.Owner == "" || p.Repo == "" || p.Number <= 0 { continue } open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) } slices.SortFunc(open, func(a, b forgemsg.OpenPull) int { if c := strings.Compare(a.Repository, b.Repository); c != 0 { return c } return a.Number - b.Number }) body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open}) if err != nil { return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err) } return Message{Event: forgemsg.OpenEvent(), Data: body}, nil }