mesh-delivery/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go

// Package bridge turns the forge module's pull request events into the messages the forge bridge writes on FORGE.
package bridge

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"maps"
	"path"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"sync"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
// redelivered.
var ErrUnreadable = errors.New("unreadable event")

// ErrNotHandled says the event is one the bridge hears but writes nothing for (yet).
var ErrNotHandled = errors.New("event not handled")

// ErrRetry says the event is readable but the forge could not be asked what the message needs; the event is to be
// heard again later.
var ErrRetry = errors.New("the forge could not say")

// Trunk is the branch a merged message is written for.
const Trunk = "main"

// Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json
// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4).
type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error)

// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448).
var ErrNotFound = errors.New("the forge has no such file")

// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge
// says is not there is ErrNotFound, any other failure any other error (issue 1448).
type Files func(owner, repo, path, ref string) ([]byte, error)

// CatalogRepository is the catalogue, whose merged message carries its predefined list, and PredefinedFile that list's
// file at its root (novox/hq ADR 0565 §2, ADR 0348 §7).
const (
	CatalogRepository = "novox/mesh-catalog"
	PredefinedFile    = "predefined.json"
)

// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465
// clause 1: go.sum for a Go module, package-lock.json for a Node one).
var LockFiles = []string{"go.sum", "package-lock.json"}

// Message is a message for FORGE: the event name (forge.<owner>.<name>.<kind>) and its JSON body.
type Message struct {
	Event string
	Data  []byte
	// Said is what the bridge did that the log says, beside writing the message; empty for none.
	Said string
}

// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea,
// published by the module as mesh.mod.<module>.event.<event>).
const (
	UpdatedSubject = "mesh.mod.gitea.event.pull.updated"
	ClosedSubject  = "mesh.mod.gitea.event.pull.closed"
	MergedSubject  = "mesh.mod.gitea.event.pull.merged"
	// DeletedSubject is the forge module's forge.delete: a branch or a tag deleted (mesh-catalog modules/gitea
	// webhook.ts: owner, repo, ref, ref_type, sha). Only the forge module's account may publish it, the subject being
	// in its own namespace (design 32 § 2), so no other module can end a branch's tagged versions.
	DeletedSubject = "mesh.mod.gitea.event.forge.delete"
)

// ForgeBranchDeleted is the kind of a branch-deleted message on FORGE (novox/hq design 51 *The forge bridge says a
// branch's end*): the builder removes the branch's tagged versions on it, as on a pull request's merged or closed.
// Every existing consumer of FORGE ignores a kind it does not know.
const ForgeBranchDeleted = "branch-deleted"

// BranchDeleted is the body of a branch-deleted message.
type BranchDeleted struct {
	V          int    `json:"v"`
	Kind       string `json:"kind"`
	Repository string `json:"repository"`
	Branch     string `json:"branch"`
	// Head is the commit the branch named when it was deleted, when the forge said it.
	Head string `json:"head,omitempty"`
}

// deleted is the part of forge.delete the bridge reads.
type deleted struct {
	Owner   string `json:"owner"`
	Repo    string `json:"repo"`
	Ref     string `json:"ref"`
	RefType string `json:"ref_type"`
	SHA     string `json:"sha"`
}

// branchDeleted turns forge.delete into a branch-deleted message; a deleted tag is not handled.
func branchDeleted(subject string, data []byte) (Message, error) {
	var d deleted
	if err := json.Unmarshal(data, &d); err != nil {
		return Message{}, unreadable(subject, "the body is not json: %v", err)
	}
	if d.RefType != "branch" {
		return Message{}, fmt.Errorf("%w: %s of a %s (%s)", ErrNotHandled, subject, d.RefType, d.Ref)
	}
	branch := strings.TrimPrefix(d.Ref, "refs/heads/")
	switch {
	case d.Owner == "" || d.Repo == "" || strings.ContainsAny(d.Owner+d.Repo, "./ *>"):
		return Message{}, unreadable(subject, "no owner or repo a forge event can name (%q/%q)", d.Owner, d.Repo)
	case branch == "":
		return Message{}, unreadable(subject, "no branch")
	case branch == Trunk:
		return Message{}, fmt.Errorf("%w: %s of the trunk %s", ErrNotHandled, subject, Trunk)
	}
	body, err := json.Marshal(BranchDeleted{V: delivery.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch, Head: d.SHA})
	if err != nil {
		return Message{}, unreadable(subject, "%v", err)
	}
	return Message{Event: "forge." + d.Owner + "." + d.Repo + "." + ForgeBranchDeleted, Data: body,
		Said: fmt.Sprintf("branch %s of %s/%s deleted: its tagged versions end", branch, d.Owner, d.Repo)}, nil
}

var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`)

// pull is the part of gitea.pull.merged (index.ts:186-207), gitea.pull.updated (mesh-catalog modules/gitea/trunk.ts:233-250) and gitea.pull.closed
// (index.ts:127-128) the bridge reads; head is the branch and head_sha the commit.
type pull struct {
	Owner          string   `json:"owner"`
	Repo           string   `json:"repo"`
	Number         int      `json:"number"`
	Head           string   `json:"head"`
	HeadSHA        string   `json:"head_sha"`
	ModuleDirs     []string `json:"module_dirs"`
	ModuleDirsSaid bool     `json:"module_dirs_said"`
	Base           string   `json:"base"`
	MergeSHA       string   `json:"merge_commit_sha"`
	// Paths are the files the pull request changes (a removed or renamed one included), PathsTruncated says the forge
	// cut the list; nil when the forge did not say them.
	Paths          *[]string `json:"paths"`
	PathsTruncated bool      `json:"paths_truncated"`
}

func unreadable(subject, why string, args ...any) error {
	return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...))
}

// Convert turns one event of the forge module, by its bus subject and body, into a message. An event it cannot read
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
// pull request left its group. A forge that cannot say is ErrRetry. A merge that leaves no module.json anywhere is a merged message with no
// trees all the same (issue 1126): the indexer takes it in and drops the repository's modules, and the view reads the
// repository as reached at it.
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
	return ConvertKept(subject, data, trees, files, nil)
}

// ConvertKept is Convert; with kept, a merge reads the forge only for the module directories whose tree hash kept
// does not hold, and keeps what it read (kept.go).
func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kept) (Message, error) {
	var action string
	switch subject {
	case UpdatedSubject:
		action = delivery.PullUpdated
	case ClosedSubject:
		action = delivery.PullClosed
	case MergedSubject:
		action = delivery.PullClosed // unless it merged into the trunk, below
	case DeletedSubject:
		return branchDeleted(subject, data)
	default:
		return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
	}
	var p pull
	if err := json.Unmarshal(data, &p); err != nil {
		return Message{}, unreadable(subject, "the body is not json: %v", err)
	}
	switch {
	case p.Owner == "" || p.Repo == "":
		return Message{}, unreadable(subject, "no owner or repo")
	case p.Number <= 0:
		return Message{}, unreadable(subject, "no pull request number")
	case !sha40.MatchString(p.HeadSHA):
		return Message{}, unreadable(subject, "the head %q is not 40 hex digits", p.HeadSHA)
	case p.Head == "":
		return Message{}, unreadable(subject, "no branch")
	}
	if subject == MergedSubject && p.Base == Trunk {
		return merged(subject, p, trees, files, kept)
	}
	said := ""
	if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
		// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
		// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
		// written closed, so a candidate an older head made is taken out, and said (issue 1422).
		action = delivery.PullClosed
		said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
			p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
	}
	f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
		Head: p.HeadSHA, Action: action}
	// The directories are the forge's word only when it said it could read them (module_dirs_said).
	if action == delivery.PullUpdated && p.ModuleDirsSaid {
		f.Dirs, f.DirsKnown = p.ModuleDirs, true
		var err error
		if f.Manifests, f.Locks, f.Needs, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
			return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
		}
	}
	event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
	if err != nil {
		return Message{}, unreadable(subject, "%v", err)
	}
	body, err := delivery.EncodeForge(f)
	if err != nil {
		return Message{}, unreadable(subject, "%v", err)
	}
	return Message{Event: event, Data: body, Said: said}, nil
}

// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
// modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event
// (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound
// (issue 1458).
const MaxReadsInFlight = 8

// read is one file asked for: the path in the directory dir, and whether it is the module's manifest.
type read struct {
	dir, path string
	manifest  bool
	asked     bool
	raw       []byte
	err       error
}

// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it
// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge
// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other
// failure is an error (issue 1448).
//
// The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
	manifests, locks, _, _, err = inputsRead(owner, repo, ref, dirs, files)
	return manifests, locks, err
}

// inputsRead is inputsOf, and also answers, on a failure too, the directories whose manifest and lock files were all
// read, with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest
// (issue 1126). It reads no mesh package a go.sum names: the indexer alone computes what a fingerprint takes (novox/hq
// ADR 0593 §2), so the message's Packages stay empty. Of a directory whose manifest names templates it reads each
// template too, in a second round, and answers the module's needs whole (catalogue.ModuleNeeds; novox/hq ADR 0632 §2):
// a template's text can need a reading its manifest does not. A template the forge does not have is no need (the judge
// refuses the manifest by name); any other failure to read one is the directory's, as a lock file's is.
func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, needs map[string][]string, whole []string, err error) {
	at := func(dir, name string) string {
		if dir == "" {
			return name
		}
		return dir + "/" + name
	}
	var reads []*read
	for _, dir := range dirs {
		reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true})
		for _, name := range LockFiles {
			reads = append(reads, &read{dir: dir, path: at(dir, name)})
		}
	}
	fetch(owner, repo, ref, reads, files)
	manifests = map[string]string{}
	locks = map[string]map[string]string{}
	broken := map[string]bool{} // a directory with a file not read (failed, or never asked after a failure)
	failed := func(r *read) bool {
		switch {
		case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
			broken[r.dir] = true
			if err == nil {
				err = fmt.Errorf("%s: %w", r.path, r.err)
			}
		case r.err == nil && r.raw == nil && !r.asked:
			broken[r.dir] = true
		default:
			return false
		}
		return true
	}
	for _, r := range reads {
		switch {
		case failed(r):
		case r.err != nil: // a lock file the forge does not have
		case r.manifest:
			manifests[r.dir] = string(r.raw)
		default:
			sum := sha256.Sum256(r.raw)
			if locks[r.dir] == nil {
				locks[r.dir] = map[string]string{}
			}
			locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
		}
	}
	var templates []*read
	if err == nil {
		for _, dir := range dirs {
			for _, t := range templatePaths(manifests[dir]) {
				templates = append(templates, &read{dir: dir, path: at(dir, t)})
			}
		}
		fetch(owner, repo, ref, templates, files)
	}
	texts := map[string]map[string]string{}
	for _, r := range templates {
		if failed(r) || r.err != nil {
			continue
		}
		if texts[r.dir] == nil {
			texts[r.dir] = map[string]string{}
		}
		texts[r.dir][strings.TrimPrefix(r.path, at(r.dir, ""))] = string(r.raw)
	}
	needs = map[string][]string{}
	for _, dir := range dirs {
		if raw, ok := manifests[dir]; ok && !broken[dir] && len(templatePaths(raw)) > 0 {
			needs[dir], _ = catalogue.ModuleNeeds([]byte(raw), texts[dir])
		}
	}
	for _, dir := range dirs {
		if !broken[dir] {
			whole = append(whole, dir)
		}
	}
	if err != nil {
		for dir := range broken {
			delete(manifests, dir)
			delete(locks, dir)
			delete(needs, dir)
		}
		return manifests, locks, needs, whole, err
	}
	if len(manifests) == 0 {
		manifests = nil
	}
	if len(locks) == 0 {
		locks = nil
	}
	if len(needs) == 0 {
		needs = nil
	}
	return manifests, locks, needs, whole, nil
}

// templatePaths is the templates a manifest names (catalogue.TemplatePaths), none for one that does not decode.
func templatePaths(raw string) []string {
	if raw == "" {
		return nil
	}
	m, err := catalogue.Decode([]byte(raw))
	if err != nil {
		return nil
	}
	return catalogue.TemplatePaths(m)
}

// fetch asks the forge for each of reads at ref, at most MaxReadsInFlight at once; after a failure (a manifest, or a
// file the forge has and could not give) no further file is asked for, and it returns once every call it started has.
func fetch(owner, repo, ref string, reads []*read, files Files) {
	var (
		wg     sync.WaitGroup
		mu     sync.Mutex
		failed bool
		slots  = make(chan struct{}, MaxReadsInFlight)
	)
	for _, r := range reads {
		slots <- struct{}{}
		mu.Lock()
		stop := failed
		mu.Unlock()
		if stop {
			<-slots
			break
		}
		r.asked = true
		wg.Add(1)
		go func(r *read) {
			defer func() { <-slots; wg.Done() }()
			r.raw, r.err = files(owner, repo, r.path, ref)
			if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) {
				mu.Lock()
				failed = true
				mu.Unlock()
			}
		}(r)
	}
	wg.Wait()
}

// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Message, error) {
	if !sha40.MatchString(p.MergeSHA) {
		return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
	}
	repository := p.Owner + "/" + p.Repo
	root, dirs, err := trees(p.Owner, p.Repo, p.MergeSHA)
	if err != nil {
		return Message{}, fmt.Errorf("%w: %s: the trees of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
	}
	names := make([]string, 0, len(dirs))
	for dir := range dirs {
		names = append(names, dir)
	}
	slices.Sort(names)
	manifests, locks, needs, toRead := keptInputs(names, dirs, kept)
	read, readLocks, readNeeds, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files)
	if err != nil {
		keep(whole, dirs, read, readLocks, readNeeds, kept) // what was read whole is not asked again at the next try
		return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, subject, repository, p.MergeSHA, len(toRead), len(names), err)
	}
	keep(toRead, dirs, read, readLocks, readNeeds, kept)
	maps.Copy(manifests, read)
	maps.Copy(locks, readLocks)
	maps.Copy(needs, readNeeds)
	if len(manifests) == 0 {
		manifests = nil
	}
	if len(locks) == 0 {
		locks = nil
	}
	if len(needs) == 0 {
		needs = nil
	}
	event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
	if err != nil {
		return Message{}, unreadable(subject, "%v", err)
	}
	f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
		Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Needs: needs}
	if repository == CatalogRepository {
		// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
		raw, err := files(p.Owner, p.Repo, PredefinedFile, p.MergeSHA)
		switch {
		case errors.Is(err, ErrNotFound):
			f.PredefinedRead = true
		case err != nil:
			return Message{}, fmt.Errorf("%w: %s: %s of %s at the merge commit %s: %v", ErrRetry, subject, PredefinedFile, repository, p.MergeSHA, err)
		default:
			f.Predefined = json.RawMessage(raw)
		}
	}
	body, err := delivery.EncodeForge(f)
	if err != nil {
		return Message{}, unreadable(subject, "%v", err)
	}
	msg := Message{Event: event, Data: body}
	if len(dirs) == 0 {
		msg.Said = fmt.Sprintf("%s at the merge commit %s holds no module.json: a merged message with no module directories", repository, p.MergeSHA)
	}
	return msg, nil
}

// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the
// branch as `head`, issue 1234/AC5), the part the bridge reads.
type openPull struct {
	Owner   string `json:"owner"`
	Repo    string `json:"repo"`
	Number  int    `json:"number"`
	Base    string `json:"base"`
	Head    string `json:"head"`
	HeadSHA string `json:"head_sha"`
}

// OpenPulls turns the answer of gitea_open_pulls, {pulls: [...]}, into one pull message per open pull request into
// the trunk, action updated and its directories not known (the forge's answer does not say them). A pull request into
// another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not
// {pulls} is an error.
func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
	var a struct {
		Pulls *[]openPull `json:"pulls"`
	}
	if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
		return nil, nil, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
	}
	for _, p := range *a.Pulls {
		if p.Base != Trunk {
			continue
		}
		name := fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number)
		if p.Owner == "" || p.Repo == "" || p.Number <= 0 || p.Head == "" || !sha40.MatchString(p.HeadSHA) {
			skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
			continue
		}
		f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
			Head: p.HeadSHA, Action: delivery.PullUpdated}
		event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
		var body []byte
		if err == nil {
			body, err = delivery.EncodeForge(f)
		}
		if err != nil {
			skipped = append(skipped, name+": "+err.Error())
			continue
		}
		msgs = append(msgs, Message{Event: event, Data: body})
	}
	return msgs, skipped, nil
}

// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
func Track(f delivery.Forge) (key, head string, ok bool) {
	key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
	switch {
	case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
		return key, f.Head, true
	case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
		return key, "", true
	}
	return "", "", false
}

// OpenMessage turns the answer of gitea_open_pulls into the open message: every open pull request into the trunk, by
// repository and number, whether its head reads or not and whether FORGE holds it or not, in repository and number
// order. A pull request into another branch is not named. An answer that names none is a message too: nothing is open.
func OpenMessage(answer []byte) (Message, error) {
	var a struct {
		Pulls *[]openPull `json:"pulls"`
	}
	if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
		return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
	}
	open := []delivery.OpenPull{}
	for _, p := range *a.Pulls {
		if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 {
			continue
		}
		open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
	}
	slices.SortFunc(open, func(a, b delivery.OpenPull) int {
		if c := strings.Compare(a.Repository, b.Repository); c != 0 {
			return c
		}
		return a.Number - b.Number
	})
	body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open})
	if err != nil {
		return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err)
	}
	return Message{Event: delivery.OpenEvent(), Data: body}, nil
}