package index import ( "context" "fmt" "maps" "slices" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) // applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that // holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest, // its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as // the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is // computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain. // // The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue // 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's // form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the // bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420 // clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. // A value the module already holds is not written again. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { digests, err := ix.keepManifests(ctx, f) if err != nil { return err } held, err := ix.heldModules(ctx) if err != nil { return err } objects, namedErr := ix.namedObjects(ctx, f) overlay := map[string]delivery.Module{} dirOf := map[string]string{} named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { raw := f.Manifests[dir] name := manifestModule(raw) if name == "" { ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir) every = false continue } key, err := delivery.ModulesKey(name) if err != nil { every = false ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err) continue } named[key] = true cur, has := held[key] if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) { ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository) continue } if other, dup := dirOf[name]; dup { if !has || cur.Path != pathOf(dir) { ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other) continue } ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir) } dirOf[name] = dir next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)} if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx)) next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) { // The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK // that reads it judges the module again at start with no merge (issue 1923). next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir]) } if in.Kept == nil && namedErr[dir] != "" { next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir] } } overlay[key] = next } // A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key // (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing. if len(f.Manifests) > 0 && every { if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil { return err } } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. return ix.recomputeWith(ctx, overlay) } // baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. func (ix *Indexer) baseFP(ctx context.Context) baseFP { return func(module string) (string, string) { key, err := delivery.ModulesKey(module) if err != nil { return "", err.Error() } value, _, found, err := ix.Modules.Get(ctx, key) if err != nil { return "", "its record cannot be read: " + err.Error() } if !found { return "", "the indexer holds no record of it" } cur, err := delivery.DecodeModule(value) switch { case err != nil: return "", "its record cannot be read: " + err.Error() case cur.FP2 != "": return cur.FP2, "" case cur.None2 != "": return "", cur.None2 } return "", "its record carries no fingerprint of the new form" } } // pathOf is the module's directory as Module.Path holds it: "." for the repository's root, since empty means not known. func pathOf(dir string) string { if dir == "" { return delivery.RootPath } return dir } func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m } // namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact // (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call // (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) { objects := map[string]map[string]string{} need := map[string][]string{} for dir, raw := range f.Manifests { m, err := catalogue.ParseManifestForInputs([]byte(raw)) if err != nil { continue // inputsOfNamed says why } paths, every := m.NamedPaths() if !every { continue } if carried := f.Paths[dir]; len(carried) > 0 { objects[dir] = carried continue } need[dir] = paths } if len(need) == 0 || ix.Named == nil { return objects, nil } got, err := ix.Named(ctx, f.Repository, f.Merge, need) if err != nil { words := map[string]string{} for dir := range need { words[dir] = err.Error() } ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err) return objects, words } for dir := range need { objects[dir] = got[dir] } return objects, nil } // dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900): // held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched. func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error { for _, key := range slices.Sorted(maps.Keys(held)) { cur := held[key] if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) { continue } err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { return nil, false, nil } if m, err := delivery.DecodeModule(value); err != nil || !delivery.SameRepository(m.Repository, repository) { return nil, false, nil } return nil, true, nil }) if err != nil { return fmt.Errorf("modules.%s: %w", key, err) } ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository) } return nil } // Reconcile deletes the modules keys whose module the trunk does not hold, for each repository the latest source names a // head of: the module directories at that head are asked of the forge (Trunk), and a module of the repository whose // directory is not among them goes (issue 1900). It is run at the indexer's start; a merge does the same from its own // message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is // and said: a failure never empties the bucket. func (ix *Indexer) Reconcile(ctx context.Context) error { if ix.Modules == nil || ix.Trunk == nil { return nil } ix.mu.Lock() heads := maps.Clone(ix.source.heads) ix.mu.Unlock() held, err := ix.heldModules(ctx) if err != nil { return err } for _, repository := range slices.Sorted(maps.Keys(heads)) { mine := map[string]delivery.Module{} for key, m := range held { if m.Repository != "" && delivery.SameRepository(m.Repository, repository) { mine[key] = m } } if len(mine) == 0 { continue } dirs, err := ix.Trunk(ctx, repository, heads[repository]) if err != nil { ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err) continue } if len(dirs) == 0 { ix.logf("%s at %s: the trunk reads as holding no module, so none is dropped", repository, heads[repository]) continue } there := map[string]bool{} for _, d := range dirs { there[pathOf(strings.Trim(d, "/"))] = true } head := heads[repository] // The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard // first) is not read as removed, only one whose commit the head is, or contains. stays := func(_ string, m delivery.Module) bool { if m.Path == "" || there[m.Path] { return true } if m.Commit == head { return false } if m.Commit == "" || ix.Contains == nil { return true } in, err := ix.Contains(ctx, repository, m.Commit, head) if err != nil { ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err) return true } return !in } if err := ix.dropRemoved(ctx, repository, mine, stays); err != nil { return err } } return nil } // None2Unreadable begins the reason of a module whose manifest the running SDK cannot read. const None2Unreadable = "the manifest cannot be read: " // unreadInputs are the inputs a merge carried for a module whose manifest cannot be read: the directory's tree, the lock // hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named // paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into // the inputs of the module (issue 1923). func unreadInputs(tree string, locks, paths map[string]string) *delivery.KeptInputs2 { if tree == "" { return nil } in := &delivery.KeptInputs2{Tree: tree} if len(locks) > 0 { in.Locks = maps.Clone(locks) } if len(paths) > 0 { in.Paths = maps.Clone(paths) } return in }