package delivery import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "reflect" "sort" "strings" ) // The modules of the parts release 3 adds (novox/hq ADR 0437): their buckets are their own state, `_`. const ( GateModule = "mesh-delivery-gate" IndexerModule = "mesh-delivery-indexer" ForgeBridgeModule = "mesh-delivery-forge-bridge" TellerModule = "mesh-delivery-teller" ) // The gate's source bucket and the indexer's candidates and modules buckets (ADR 0437 §5). The source bucket has // one key, SourceMainKey: every repository's merged head and the group that moved it, written by the gate alone by // compare-and-set; its revision is the mesh's source revision. The indexer writes the other two. const ( SourceBucket = GateModule + "_source" SourceMainKey = "main" SourceHistory = 64 CandidatesBucket = IndexerModule + "_candidates" // CandidatesHistory is the revisions the candidates bucket keeps per key. CandidatesHistory = 10 ModulesBucket = IndexerModule + "_modules" // ModulesHistory is the revisions the modules bucket keeps per key. ModulesHistory = 20 // ManifestsBucket is the indexer's bucket of manifests by digest (novox/hq ADR 0429 clause 16); see manifests.go. ManifestsBucket = IndexerModule + "_manifests" // ManifestsHistory is the revisions the manifests bucket keeps per key: a key is written once. ManifestsHistory = 1 ) // ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names. const ForgeStream = "forge" // The kinds of forge message, the last part of a forge event's name. const ( ForgePull = "pull" ForgeMerged = "merged" ForgeRefused = "refused" ForgeOpen = "open" ) // OpenPull names an open pull request into the trunk: the repository as `/` and the number. type OpenPull struct { Repository string `json:"repository"` Number int `json:"number"` } // OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of // every repository the forge bridge reads (novox/hq ADR 0443 §5). func OpenEvent() string { return "forge." + ForgeOpen } // The actions of a pull message. const ( PullOpened = "opened" PullUpdated = "updated" PullClosed = "closed" ) // SourcePrefix opens every source fingerprint. const SourcePrefix = "src1:" // CandidatesKey is the key, in CandidatesBucket, of a group: ``, the branch name the group's pull requests // share (a slash is allowed); a lone pull request is a group of one under its branch name. func CandidatesKey(group string) (string, error) { return groupKey(group) } // ParseCandidatesKey is the inverse of CandidatesKey. func ParseCandidatesKey(key string) (group string, err error) { if group, err = parseGroupKey(key); err != nil { return "", fmt.Errorf("delivery: %q is not a candidates key: %w", key, err) } return group, nil } // groupKey is a group name as a key part: escaped (EscapeName), so a branch such as `fix/1.2` keys. func groupKey(group string) (string, error) { if group == "" { return "", fmt.Errorf("delivery: the group name is empty") } return EscapeName(group), nil } // parseGroupKey is the inverse of groupKey. func parseGroupKey(part string) (string, error) { if part == "" { return "", fmt.Errorf("delivery: the group name is empty") } group, err := UnescapeName(part) if err != nil { return "", err } return group, nil } // escapeRepository is `/` as a key part, each part escaped; the repository must have exactly one slash // and neither part empty. func escapeRepository(repository string) (string, error) { owner, name, ok := strings.Cut(repository, "/") if !ok || owner == "" || name == "" || strings.Contains(name, "/") { return "", fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) } return EscapeName(owner) + "/" + EscapeName(name), nil } // unescapeRepository is the inverse of escapeRepository. func unescapeRepository(part string) (string, error) { owner, name, ok := strings.Cut(part, "/") if !ok || owner == "" || name == "" || strings.Contains(name, "/") { return "", fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", part) } o, err := UnescapeName(owner) if err != nil { return "", err } n, err := UnescapeName(name) if err != nil { return "", err } return o + "/" + n, nil } // ModulesKey is the key, in ModulesBucket, of a module: ``. func ModulesKey(module string) (string, error) { if err := checkName("module", module); err != nil { return "", err } return module, nil } // ParseModulesKey is the inverse of ModulesKey. func ParseModulesKey(key string) (module string, err error) { if module, err = ModulesKey(key); err != nil { return "", fmt.Errorf("delivery: %q is not a modules key: %w", key, err) } return module, nil } // checkRepository refuses a repository that is not `/`: exactly one slash, each part a valid name. func checkRepository(repository string) error { owner, name, ok := strings.Cut(repository, "/") if !ok || strings.Contains(name, "/") { return fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) } if err := checkName("repository owner", owner); err != nil { return fmt.Errorf("delivery: the repository %q: %w", repository, err) } if err := checkName("repository name", name); err != nil { return fmt.Errorf("delivery: the repository %q: %w", repository, err) } if strings.Contains(owner, "/") { return fmt.Errorf("delivery: the repository %q is not /, with exactly one slash", repository) } return nil } func checkForgeKind(kind string) error { switch kind { case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen: return nil } return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind) } // ForgeEvent is the name of a forge message of the given kind: `forge...`. func ForgeEvent(repository, kind string) (string, error) { if err := checkForgeKind(kind); err != nil { return "", err } if kind == ForgeOpen { return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent()) } if err := checkRepository(repository); err != nil { return "", err } owner, name, _ := strings.Cut(repository, "/") return "forge." + owner + "." + name + "." + kind, nil } // ParseForgeEvent is the inverse of ForgeEvent. func ParseForgeEvent(event string) (repository, kind string, err error) { if event == OpenEvent() { return "", ForgeOpen, nil } parts := strings.Split(event, ".") if len(parts) != 4 || parts[0] != "forge" { return "", "", fmt.Errorf("delivery: %q is not a forge event, forge...", event) } repository = parts[1] + "/" + parts[2] if _, err := ForgeEvent(repository, parts[3]); err != nil { return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err) } return repository, parts[3], nil } // Head is one pull request at one commit: the repository as `/` and the full sha. type Head struct { Repository string `json:"repository"` Number int `json:"number"` Head string `json:"head"` } // HeadsHash is the first 12 hex digits of the sha256 over the lines `#=`, sorted and // joined by a newline: the same for every order of heads, different when any head changes. An empty list is refused. func HeadsHash(heads []Head) (string, error) { if len(heads) == 0 { return "", fmt.Errorf("delivery: a heads hash needs at least one pull request") } lines := make([]string, len(heads)) for i, h := range heads { lines[i] = fmt.Sprintf("%s#%d=%s", h.Repository, h.Number, h.Head) } sort.Strings(lines) sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) return hex.EncodeToString(sum[:])[:12], nil } // Source is the value of SourceMainKey: every repository's merged head (repository to full sha, not only the // group's) and the group that moved it. type Source struct { V int `json:"v"` Group string `json:"group"` Heads map[string]string `json:"heads"` } func (s Source) check() error { if len(s.Heads) == 0 { return fmt.Errorf("delivery: a source names no repository") } for repository, head := range s.Heads { if repository == "" { return fmt.Errorf("delivery: a source names a repository with no name") } if head == "" { return fmt.Errorf("delivery: the source's head of %s is empty", repository) } } return nil } // Candidate is the value of a candidates key: the group's pull requests and heads, the heads hash, the base (the // source revision it was previewed onto) and the modules each pull request touches. It carries no fingerprint // until release 4 (ADR 0437 §5); from R4c the indexer writes the new form's fingerprint of each touched module in // the pull request's Fps and, over all of them, FPSet (novox/hq ADR 0453 clauses 6 and 16). Both are absent in a // value written before, and a reader of today's shape ignores them. type Candidate struct { V int `json:"v"` Group string `json:"group"` Base uint64 `json:"base"` HeadsHash string `json:"heads"` Pulls []CandidatePull `json:"pulls"` FPSet string `json:"fpset,omitempty"` } // CandidatePull is one pull request of a candidate and the modules it touches. Fps is, by module name, the new // form's fingerprint of the module at the merge preview, or why it has none, and the digest of its manifest (the key // of the indexer's manifests bucket); a module of Modules without an entry has none written yet. type CandidatePull struct { Head Modules []string `json:"modules"` Fps map[string]CandidateModule `json:"fps,omitempty"` } // CandidateModule is one touched module's new-form fingerprint FP2 (`src2:` and 64 hex digits), or None2, the reason // it has none; exactly one of them. Manifest is the digest of the module's manifest as read at the preview. type CandidateModule struct { FP2 string `json:"fp2,omitempty"` None2 string `json:"none2,omitempty"` Manifest string `json:"manifest,omitempty"` } func (m CandidateModule) check() error { switch { case m.FP2 != "" && m.None2 != "": return fmt.Errorf("delivery: a module has both fp2 and none2") case m.FP2 == "" && m.None2 == "": return fmt.Errorf("delivery: a module has neither fp2 nor none2") } return checkNewForm(m.FP2, m.Manifest) } // checkNewForm refuses an fp2 that is not of the new form and a manifest digest that is not a digest, naming it. func checkNewForm(fp2, manifest string) error { if fp2 != "" { if _, err := FingerprintKeyPart(fp2); err != nil { return err } } if manifest != "" { if _, err := ManifestsKey(manifest); err != nil { return err } } return nil } // FingerprintSet is the fpset of a candidate's pull requests: the first 12 hex digits of the sha256 over the lines // `=` or `=none:` of every module with an entry in Fps, sorted and joined by a newline, // so it is the same for every order and changes with any fingerprint or reason. A manifest digest does not enter: the // fingerprint already covers the manifest. It is the empty string when no pull request carries a fingerprint. The // checker keys its verdict by it (research 063 section 01). func FingerprintSet(pulls []CandidatePull) string { lines := map[string]bool{} for _, p := range pulls { for module, m := range p.Fps { if m.FP2 != "" { lines[module+"="+m.FP2] = true } else { lines[module+"=none:"+m.None2] = true } } } if len(lines) == 0 { return "" } all := make([]string, 0, len(lines)) for l := range lines { all = append(all, l) } sort.Strings(all) sum := sha256.Sum256([]byte(strings.Join(all, "\n"))) return hex.EncodeToString(sum[:])[:12] } func (c Candidate) check() error { if len(c.Pulls) == 0 { return fmt.Errorf("delivery: a candidate names no pull request") } heads := make([]Head, len(c.Pulls)) for i, p := range c.Pulls { heads[i] = p.Head } want, _ := HeadsHash(heads) if c.HeadsHash != want { return fmt.Errorf("delivery: the candidate's heads hash %q is not the hash %q of its pull requests' heads", c.HeadsHash, want) } seen := map[string]CandidateModule{} for _, p := range c.Pulls { touched := map[string]bool{} for _, m := range p.Modules { touched[m] = true } for module, m := range p.Fps { if !touched[module] { return fmt.Errorf("delivery: the candidate gives a fingerprint of the module %q, which pull request %s#%d does not touch", module, p.Repository, p.Number) } if err := m.check(); err != nil { return fmt.Errorf("delivery: the candidate's module %q: %w", module, err) } if before, ok := seen[module]; ok && before != m { return fmt.Errorf("delivery: two pull requests of the candidate give the module %q different fingerprints", module) } seen[module] = m } } if wantSet := FingerprintSet(c.Pulls); c.FPSet != wantSet { return fmt.Errorf("delivery: the candidate's fpset %q is not the hash %q of its fingerprints", c.FPSet, wantSet) } return nil } // Module is the value of a modules key: the module's source fingerprint, or none and why (a build that resolves // packages when built gets none, said here), and where it was computed: Repository and Commit are the trunk merge, // Src the source revision then (0 before the gate writes source.main). Exactly one of FP and None is set. // // From R4c the indexer writes the new form beside it (novox/hq ADR 0453 clause 6): FP2, the fingerprint of the new // form (`src2:` and 64 hex digits), or None2, why it has none; one of them whenever either is, never both; and // Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of // the three, and a reader of today's shape ignores them. type Module struct { V int `json:"v"` FP string `json:"fp,omitempty"` None string `json:"none,omitempty"` FP2 string `json:"fp2,omitempty"` None2 string `json:"none2,omitempty"` Manifest string `json:"manifest,omitempty"` // Inputs2 are the new-form inputs the merge gave the module, kept so the indexer computes FP2 again when a build // module's toolchain versions appear, with no merge (novox/hq ADR 0465 clause 2). It needs Manifest. Inputs2 *KeptInputs2 `json:"inputs2,omitempty"` Src uint64 `json:"src,omitempty"` Repository string `json:"repository"` Commit string `json:"commit"` // Path is the module's directory in its repository, trimmed of slashes, and RootPath (".") when its manifest is the // repository's root (novox/hq ADR 0478 clause 9, issue 1615): the build tools need it. Empty means not known: a value // written before the indexer learned paths has none. Path string `json:"path,omitempty"` // Toolchains are the toolchains FP2 was computed with, by language (go, typescript): the version installed and the // build module's recipe version the build nodes agreed on then (novox/hq ADR 0477 clause 2, ADR 0478 clause 3). A builder // builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written // before has none (issue 1615). Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` } // RootPath is Module.Path of a module whose manifest is its repository's root. const RootPath = "." // KeptInputs2 are SourceInputs2 without the module's name (the key has it) and without toolchains (those come from // the build modules' toolchain keys when the fingerprint is computed): what a Module keeps of a merge. type KeptInputs2 struct { Tree string `json:"tree"` Bases []string `json:"bases,omitempty"` Builder string `json:"builder,omitempty"` Contexts map[string]string `json:"contexts,omitempty"` Locks map[string]string `json:"locks,omitempty"` } // With is the full inputs of module m: the kept ones and the given toolchains. func (k KeptInputs2) With(module string, toolchains map[string]Toolchain2) SourceInputs2 { return SourceInputs2{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts, Locks: k.Locks, Toolchains: toolchains} } func (m Module) check() error { switch { case m.FP != "" && m.None != "": return fmt.Errorf("delivery: a module has both a fingerprint and a reason for none") case m.FP == "" && m.None == "": return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none") case m.FP != "" && !strings.HasPrefix(m.FP, SourcePrefix): return fmt.Errorf("delivery: the module's fingerprint %q does not start with %s", m.FP, SourcePrefix) case m.FP2 != "" && m.None2 != "": return fmt.Errorf("delivery: a module has both fp2 and none2") } for language, t := range m.Toolchains { if t.Version == "" || t.Recipe == "" { return fmt.Errorf("delivery: the module's toolchain of %q needs its version and its recipe version", language) } } if m.Inputs2 != nil && m.Manifest == "" { return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest") } return checkNewForm(m.FP2, m.Manifest) } // Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull // message says Action; a merged message carries the merge commit, the tree of every directory holding a // module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own // words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and // DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither // (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither. type Forge struct { V int `json:"v"` Kind string `json:"kind"` Repository string `json:"repository"` Branch string `json:"branch,omitempty"` Number int `json:"number,omitempty"` Head string `json:"head,omitempty"` Action string `json:"action,omitempty"` Dirs []string `json:"dirs,omitempty"` DirsKnown bool `json:"dirs-known,omitempty"` Merge string `json:"merge,omitempty"` Trees map[string]string `json:"trees,omitempty"` Root string `json:"root,omitempty"` // Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message, // of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. Manifests map[string]string `json:"manifests,omitempty"` Locks map[string]map[string]string `json:"locks,omitempty"` Act string `json:"act,omitempty"` Words string `json:"words,omitempty"` Open []OpenPull `json:"open,omitempty"` } func (f Forge) check() error { if err := checkForgeKind(f.Kind); err != nil { return err } if f.Kind == ForgeOpen { return f.checkOpen() } if len(f.Open) > 0 { return fmt.Errorf("delivery: only an open message names open pull requests") } if err := checkRepository(f.Repository); err != nil { return err } switch f.Kind { case ForgePull: if f.Number <= 0 { return fmt.Errorf("delivery: a pull message needs its pull request number") } if f.Head == "" { return fmt.Errorf("delivery: a pull message needs its head") } switch f.Action { case PullOpened, PullUpdated, PullClosed: default: return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action) } if len(f.Dirs) > 0 && !f.DirsKnown { return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)") } dirs := map[string]bool{} for _, d := range f.Dirs { dirs[d] = true } if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil { return err } case ForgeMerged: if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a merged message carries no module directories") } if f.Number <= 0 { return fmt.Errorf("delivery: a merged message needs its pull request number") } if f.Head == "" { return fmt.Errorf("delivery: a merged message needs its head") } if f.Merge == "" { return fmt.Errorf("delivery: a merged message needs its merge commit") } if f.Root == "" { return fmt.Errorf("delivery: a merged message needs the repository's root tree") } if len(f.Trees) == 0 { return fmt.Errorf("delivery: a merged message needs the tree of at least one directory holding a module.json") } for dir, tree := range f.Trees { if tree == "" { return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir) } } if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil { return err } case ForgeRefused: if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a refused message carries no module directories") } if len(f.Manifests) > 0 || len(f.Locks) > 0 { return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes") } if f.Act == "" { return fmt.Errorf("delivery: a refused message needs the act it answers") } if f.Words == "" { return fmt.Errorf("delivery: a refused message needs the forge's words") } } return nil } // checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them // by), an empty manifest and an empty lock path or hash, naming the directory. func (f Forge) checkInputs(known func(dir string) bool, of string) error { for dir, m := range f.Manifests { if !known(dir) { return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of) } if m == "" { return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir) } } for dir, locks := range f.Locks { if !known(dir) { return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of) } for path, hash := range locks { if path == "" || hash == "" { return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir) } } } return nil } // checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or // an open pull request that is not `/` and a number. func (f Forge) checkOpen() error { if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown || f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { return fmt.Errorf("delivery: an open message carries only the open pull requests it names") } for _, p := range f.Open { if err := checkRepository(p.Repository); err != nil { return err } if p.Number <= 0 { return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository) } } return nil } // EncodeSource is s as JSON at Version; it refuses a source with no repository or an empty head. func EncodeSource(s Source) ([]byte, error) { if err := s.check(); err != nil { return nil, err } s.V = Version return json.Marshal(s) } // DecodeSource reads a source; it refuses an unknown version (ErrUnknownVersion) and what EncodeSource refuses. func DecodeSource(data []byte) (Source, error) { var s Source if err := decode(data, &s); err != nil { return Source{}, err } if err := s.check(); err != nil { return Source{}, err } return s, nil } // EncodeCandidate is c as JSON at Version; it refuses a heads hash that is not the hash of its pull requests' heads. func EncodeCandidate(c Candidate) ([]byte, error) { if err := c.check(); err != nil { return nil, err } c.V = Version return json.Marshal(c) } // DecodeCandidate reads a candidate; it refuses an unknown version (ErrUnknownVersion) and what EncodeCandidate refuses. func DecodeCandidate(data []byte) (Candidate, error) { var c Candidate if err := decode(data, &c); err != nil { return Candidate{}, err } if err := c.check(); err != nil { return Candidate{}, err } return c, nil } // EncodeModule is m as JSON at Version; it refuses both or neither of a fingerprint and a reason, and a // fingerprint not of the form src1:. func EncodeModule(m Module) ([]byte, error) { if err := m.check(); err != nil { return nil, err } m.V = Version return json.Marshal(m) } // DecodeModule reads a module; it refuses an unknown version (ErrUnknownVersion) and what EncodeModule refuses. func DecodeModule(data []byte) (Module, error) { var m Module if err := decode(data, &m); err != nil { return Module{}, err } if err := m.check(); err != nil { return Module{}, err } return m, nil } // EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs. func EncodeForge(f Forge) ([]byte, error) { if err := f.check(); err != nil { return nil, err } f.V = Version return json.Marshal(f) } // DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses. func DecodeForge(data []byte) (Forge, error) { var f Forge if err := decode(data, &f); err != nil { return Forge{}, err } if err := f.check(); err != nil { return Forge{}, err } return f, nil } // SameSource says whether two sources are the same value, so a writer writes only when they differ. func SameSource(a, b Source) bool { return reflect.DeepEqual(a, b) } // SameCandidate says whether two candidates are the same value. func SameCandidate(a, b Candidate) bool { return reflect.DeepEqual(a, b) } // SameModule says whether two modules are the same value. func SameModule(a, b Module) bool { return reflect.DeepEqual(a, b) } // SameForge says whether two forge messages are the same value. func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }