package index import ( "context" "encoding/json" "errors" "fmt" "maps" "path" "slices" "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" ) // The gate's source revision of a watched branch (design 47 G §15, §115), read through the seat mesh-delivery's verb source // and its event source-moved, never the gate's state: it names every repository's merged head, and its revision is the // mesh's source revision of that branch. The indexer keeps the latest it has seen. A module is stamped // with it (Module.Src) when the head the source names for its repository is the merge commit the module's fingerprint // was computed at: from the modules bucket when the merge was heard first, and when the merge is heard after the source. // sourceNow is the latest source revision seen. type sourceNow struct { rev uint64 heads map[string]string } // srcOf is the source revision that names this merge commit as the repository's head, or 0 when the latest source seen // names another head or none has been seen. func (ix *Indexer) srcOf(repository, merge string) uint64 { ix.mu.Lock() defer ix.mu.Unlock() if ix.source.rev != 0 && merge != "" && ix.source.heads[repository] == merge { return ix.source.rev } return 0 } // sourceHeads is the head the latest source seen names for each repository (a copy), nil while none has been seen. func (ix *Indexer) sourceHeads() map[string]string { ix.mu.Lock() defer ix.mu.Unlock() return maps.Clone(ix.source.heads) } // ApplySource takes the source revision the verb source answered in. An older revision than the latest seen is left. For each // repository the source names, each module built from it whose fingerprint was computed at the head the source names, and // that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped // when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The // modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6). func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) error { rev := uint64(a.Revision) s := struct{ Heads map[string]string }{Heads: map[string]string{}} for _, r := range a.Repositories { s.Heads[r.Repository] = r.Head } ix.kickJudge(ctx) ix.mu.Lock() if rev < ix.source.rev { ix.mu.Unlock() return nil } before := ix.source.heads ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)} ix.mu.Unlock() if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } if err := ix.kickCatchUp(ctx, s.Heads); err != nil { return err } // A registered module the forge could not read is read again at each new source revision (issue 1945), in the // background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written // (applyManifests reads srcOf). ix.kickRetry(ctx) held, err := ix.heldModules(ctx) if err != nil { return err } for _, key := range slices.Sorted(maps.Keys(held)) { repository := "" for r := range s.Heads { if trunk.SameRepository(r, held[key].Repository) { repository = r } } if repository == "" { continue } head := s.Heads[repository] err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { return nil, false, nil } cur, err := meshcatalogclient.DecodeModule(value) if err != nil { ix.logf("modules.%s holds what is not a module, left: %v", key, err) return nil, false, nil } // Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another // repository) leaves the value as it is, so no revision of the source rewrites a module that did not change. if cur.Repository != repository || cur.Commit != head || cur.Src != 0 { return nil, false, nil } cur.Src = int64(rev) if cur.FpSrc == 0 { cur.FpSrc = int64(rev) // its fingerprint moved at this merge, whose revision is heard only now (ADR 0601 §1) } data, err := meshcatalogclient.EncodeModule(cur) return data, false, err }) if err != nil { return fmt.Errorf("modules.%s: %w", key, err) } } // A module built against the commit the source named as a repository's head (a package no module publishes, A-365) is // computed again when that head moves: the gate moves the source only after the take-in. if movedPackages(held, before, s.Heads) { if err := ix.Recompute(ctx); err != nil { return err } } // The stamps are kept with the view (KeepView), so a restart reads them back. if err := ix.KeepView(ctx); err != nil { return err } // A trunk that moved makes the previews made onto the old one out of date. return ix.RecomputeCandidates(ctx) } // movedPackages is whether a held module's package version is the previous head of a repository whose head the new source moved. func movedPackages(held map[string]meshcatalog.ModulesByModuleValue, before, now map[string]string) bool { for _, m := range held { for p, v := range m.Packages { repo, _, ok := trunk.PackageRepository(p) if ok && before[repo] != "" && before[repo] == v && now[repo] != before[repo] { return true } } } return false } // kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on // milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge // bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision // the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said // when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its // bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is // not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus // yet run none, said, until then. func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error { if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 { return nil } if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects) return nil } else if err != nil || any { return err } if !ix.catchingUp.CompareAndSwap(false, true) { return nil } heads = maps.Clone(heads) ix.catchUpWG.Add(1) go func() { defer ix.catchUpWG.Done() unread := ix.takeInHeads(ctx, heads) ix.catchingUp.Store(false) if len(unread) > 0 { ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision", len(unread), strings.Join(unread, ", ")) return } if err := ix.KeepView(ctx); err != nil { ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err) return } ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads)) }() return nil } // takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the // repositories whose message was not read or not applied. func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string { var unread []string for _, repository := range slices.Sorted(maps.Keys(heads)) { head := heads[repository] if head == "" { continue } f, err := ix.mergedAt(ctx, repository, head) if err != nil && ctx.Err() == nil && ix.Read != nil { // A head the source names that has no merged message (a branch newly watched, whose head no merge on it made) is // read whole at that commit, as a clone does (A-401, A-93). ix.logf("catalog graph: %s at %.12s has no merged message (%v): read whole at that commit through the seat git, as a merge of every file of its tree", repository, head, err) f, err = ix.wholeAt(ctx, repository, head) } if err == nil { _, err = ix.ApplyMessage(ctx, f) } if err != nil { unread = append(unread, repository) ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err) } } return unread } // CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its // manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it // with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first // start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry // moved. Its answer names the repositories taken in and those whose merged message could not be read. func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { out := TakenIn{Module: "catch-up"} ix.mu.Lock() heads := maps.Clone(ix.source.heads) ix.mu.Unlock() switch { case len(heads) == 0: out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes" return out case ix.Modules == nil || ix.Message == nil: out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in" return out case !ix.catchingUp.CompareAndSwap(false, true): out.Answer = "a catch-up runs already; its end keeps the revision" return out } unread := ix.takeInHeads(ctx, heads) ix.catchingUp.Store(false) if err := ix.KeepView(ctx); err != nil { out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error() return out } out.Stored = len(unread) == 0 out.Unread = unread out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread)) if len(unread) > 0 { out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", ")) } return out } // mergedAt is the forge bridge's merged message of a repository's head. func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { data, err := ix.Message(forTakeIn(ctx, repository, head), repository, head, 0) if err != nil { return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err) } f, err := forgemsg.DecodeForge(data) if err != nil { return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err) } if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) { return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head) } return f, nil } // wholeAt is the merged message of a head that has none (A-401): the head's tree and files read through the seat git // (Read), every module directory of it with its manifest, lock files and mesh packages, as a merge whose changed files are // every file of that tree, so it is computed as any other head (applyMoving; A-93). func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: head, Head: head} file := func(p string) ([]byte, error) { if ix.ReadFile == nil { return nil, errors.New("the indexer reads no file of the forge") } return ix.ReadFile(ctx, repository, head, p) } first, err := ix.Read(ctx, repository, head, "") if err != nil { return f, fmt.Errorf("the seat git could not read %s at %.12s: %w", repository, head, err) } f.Root, f.Trees = first.Root, maps.Clone(first.Trees) for _, dir := range slices.Sorted(maps.Keys(first.Trees)) { read := first if dir != "" { if read, err = ix.Read(ctx, repository, head, dir); err != nil { return f, fmt.Errorf("the seat git could not read %q of %s at %.12s: %w", dir, repository, head, err) } } if f.Manifests == nil { f.Manifests = map[string]string{} } f.Manifests[dir] = read.Manifests[dir] // The reader capabilities of a manifest that names templates, its templates' text too, as the forge bridge reads them // (ADR 0632 §2): a template the forge does not have is no need, any other failure to read one is the head's. if paths := manifestTemplatePaths(read.Manifests[dir]); len(paths) > 0 { texts := map[string]string{} for _, t := range paths { content, err := file(path.Join(dir, t)) if err != nil { return f, fmt.Errorf("the seat git could not read the template %q of %q of %s at %.12s: %w", t, dir, repository, head, err) } if content != nil { texts[t] = string(content) } } if f.Needs == nil { f.Needs = map[string][]string{} } f.Needs[dir], _ = catalogue.ModuleNeeds([]byte(read.Manifests[dir]), texts) } if l := read.Locks[dir]; len(l) > 0 { if f.Locks == nil { f.Locks = map[string]map[string]string{} } f.Locks[dir] = l } if p := read.Packages[dir]; len(p) > 0 { if f.Packages == nil { f.Packages = map[string][]string{} } f.Packages[dir] = p } } // The catalogue's predefined list at the commit, as the forge bridge carries it (ADR 0565 §2). if trunk.SameRepository(repository, catalogueRepository) { raw, err := file(predefinedFile) if err != nil { return f, fmt.Errorf("the seat git could not read %s of %s at %.12s: %w", predefinedFile, repository, head, err) } if raw == nil { f.PredefinedRead = true } else { f.Predefined = json.RawMessage(raw) } } return f, nil } // The catalogue repository and its predefined list's file, whose merged message carries the list (the forge bridge's // CatalogRepository and PredefinedFile). const ( catalogueRepository = "novox/mesh-catalog" predefinedFile = "predefined.json" ) // manifestTemplatePaths is the template paths a manifest names, none for one that cannot be read. func manifestTemplatePaths(raw string) []string { m, err := catalogue.Decode([]byte(raw)) if err != nil { return nil } return catalogue.TemplatePaths(m) }