From a75d9cc851212e7a1bae9ee027b38830284002d5 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:54:28 +0000 Subject: [PATCH] Keep manifests and say why no new-form fingerprint exists, so the group check can start from facts (issues 1449, 1450, 1444, 1451, design 47 R4c-2) --- .../internal/index/candidates.go | 67 ++++++++-- .../internal/index/inputs.go | 115 ++++++++++++++++++ .../internal/index/manifests.go | 83 +++++++++++++ .../internal/index/modules.go | 52 +++++++- 4 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/inputs.go create mode 100644 modules/mesh-delivery-indexer/internal/index/manifests.go diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 7be504a..747700c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -43,8 +43,10 @@ type Indexer struct { Store Store // candidates // Modules is the modules bucket; nil writes no fingerprints. Modules Store - Builds *Builds - Logf func(format string, args ...any) + // Manifests is the manifests bucket; nil writes no manifests (issue 1449). + Manifests Store + Builds *Builds + Logf func(format string, args ...any) mu sync.Mutex // guards source, which the source watch and the forge messages both reach source sourceNow @@ -72,6 +74,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { return err } } + var digests map[string]string + if f.Kind == delivery.ForgePull { + var err error + if digests, err = ix.keepManifests(ctx, f); err != nil { + return err + } + } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) return nil @@ -89,7 +98,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } cur = &c } - next := ix.next(cur, f) + next := ix.next(cur, f, digests) + if next != nil { + if err := ix.refuseDisagreement(*next); err != nil { + return nil, false, err + } + } switch { case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next): return nil, false, nil @@ -185,7 +199,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, } // next is the group after the message; nil when the group is left with no pull request. -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate { +// +// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of +// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a +// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by +// directory. +func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { @@ -202,13 +221,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can switch { case f.DirsKnown: for _, dir := range f.Dirs { - if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) { + m := ix.Builds.ModuleOf(f.Repository, dir) + if !slices.Contains(entry.Modules, m) { entry.Modules = append(entry.Modules, m) } + if digest, ok := digests[dir]; ok { + if entry.Fps == nil { + entry.Fps = map[string]delivery.CandidateModule{} + } + entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview} + } } case at >= 0 && pulls[at].Head.Head == f.Head: // Its directories are not known now, for the head the group already holds: what it knew stands. - entry.Modules = pulls[at].Modules + entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps } if at >= 0 { pulls[at] = entry @@ -230,5 +256,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can heads[i] = p.Head } hash, _ := delivery.HeadsHash(heads) - return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls} + return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)} +} + +// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not +// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445). +const NoneNoPreview = "no merge preview is computed" + +// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the +// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is +// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull +// request is judged again. +func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error { + first := map[string]delivery.CandidatePull{} + for _, p := range c.Pulls { + for module, m := range p.Fps { + before, ok := first[module] + if !ok { + first[module] = p + continue + } + if before.Fps[module] != m { + err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", + ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) + return err + } + } + } + return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go new file mode 100644 index 0000000..52f189b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -0,0 +1,115 @@ +package index + +import ( + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq +// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport. +const ( + // None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge). + None2NoManifest = "the message carries no manifest" + // None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing. + None2Builder = "builder's fingerprint not yet known" + // None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none + // either, NoneContext). + None2Context = "context not pinned" +) + +// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's +// directory (the bridge reads these, internal/bridge LockFiles; issue 1450). +var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"} + +// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450). +type moduleInputs struct { + // Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none. + Kept *delivery.KeptInputs2 + // Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by + // artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the + // watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then. + Languages map[string]string + // None2 is why the module has no fingerprint of the new form. + None2 string +} + +// baseFP answers a base module's new-form fingerprint, or why it has none. +type baseFP func(module string) (fp, none string) + +// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock +// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's +// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why. +// +// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a +// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages, +// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its +// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive, +// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module. +func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} + } + if m.Module != module { + return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)} + } + if tree == "" { + return moduleInputs{None2: "the merge names no tree for the module's directory"} + } + kept := delivery.KeptInputs2{Tree: tree} + langs := map[string]string{} + contexts := false + var missing []string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if a.Context != nil { + contexts = true + } + if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage { + continue + } + name, ok := lockFile[a.Language] + if !ok { + continue + } + langs[a.Name] = a.Language + if kept.Locks == nil { + kept.Locks = map[string]string{} + } + kept.Locks[a.Name] = locks[name] + } + seen := map[string]bool{} + for _, on := range m.Build.On { + if on.Module == "" || seen[on.Module] { + continue + } + seen[on.Module] = true + if fp, none := base(on.Module); fp != "" { + kept.Bases = append(kept.Bases, fp) + } else { + missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none)) + } + } + } + sort.Strings(kept.Bases) + sort.Strings(missing) + out := moduleInputs{Kept: &kept} + if len(langs) > 0 { + out.Languages = langs + } + switch { + case contexts: + out.None2 = None2Context + case len(missing) > 0: + out.None2 = strings.Join(missing, "; ") + case len(langs) > 0: + out.None2 = delivery.ToolchainNoReport + default: + out.None2 = None2Builder + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go new file mode 100644 index 0000000..d1883b1 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -0,0 +1,83 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests +// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the +// module a build record names the directory the path of. +func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string { + var m struct { + Module string `json:"module"` + } + if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" { + return m.Module + } + name, _ := ix.Builds.Module(repository, dir) + return name +} + +// keepManifest writes manifests. for a manifest as the forge answered it, create-once (issue 1449): a key that +// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for +// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the +// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried +// again. With no manifests store nothing is written and the digest is still answered. +func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) { + digest := delivery.ManifestDigest([]byte(raw)) + if ix.Manifests == nil { + return digest, nil + } + key, err := delivery.ManifestsKey(digest) + if err != nil { + return "", err + } + data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + if err != nil { + return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err) + } + err = ix.Manifests.Create(ctx, key, data) + if err == nil { + return digest, nil + } + if !errors.Is(err, ErrConflict) { + return "", fmt.Errorf("writing manifests.%s: %w", key, err) + } + held, _, found, err := ix.Manifests.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("reading manifests.%s: %w", key, err) + } + if !found { + return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key) + } + if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) { + return digest, nil + } + return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module) +} + +// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory. +// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest. +func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { + digests := make(map[string]string, len(f.Manifests)) + for dir, raw := range f.Manifests { + module := ix.moduleNameOf(f.Repository, dir, raw) + digest, err := ix.keepManifest(ctx, module, raw) + switch { + case err == nil: + digests[dir] = digest + case errors.Is(err, errCannotBeAValue): + ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err) + default: + return nil, err + } + } + return digests, nil +} + +var errCannotBeAValue = errors.New("not a manifest value") diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 1243ff5..753df87 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -3,6 +3,7 @@ package index import ( "context" "fmt" + "reflect" "sort" "strings" @@ -47,8 +48,14 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) { // applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and // why), with the source revision that names the merge commit as the repository's head when the indexer has seen it // (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already -// holds is not written again. +// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it +// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, +// 1450); this release writes no fingerprint of the new form for any module. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { + digests, err := ix.keepManifests(ctx, f) + if err != nil { + return err + } for _, k := range ix.Builds.Modules(f.Repository) { key, err := delivery.ModulesKey(k.Module) if err != nil { @@ -56,10 +63,22 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { continue } fp, none := fingerprintOf(k, f.Trees) - next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge)} + next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path]} + // The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which + // names what is missing, from its manifest and lock hashes (inputs.go). + raw, has := f.Manifests[k.Path] + switch { + case !has: + next.None2 = None2NoManifest + case next.Manifest == "": + next.None2 = "the manifest cannot be read: it cannot be kept as a value" + default: + in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) + next.Inputs2, next.None2 = in.Kept, in.None2 + } err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if found { - if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) { + if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) { return nil, false, nil } } @@ -73,4 +92,31 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { return nil } +// baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. +func (ix *Indexer) baseFP(ctx context.Context) baseFP { + return func(module string) (string, string) { + key, err := delivery.ModulesKey(module) + if err != nil { + return "", err.Error() + } + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return "", "its record cannot be read: " + err.Error() + } + if !found { + return "", "the indexer holds no record of it" + } + cur, err := delivery.DecodeModule(value) + switch { + case err != nil: + return "", "its record cannot be read: " + err.Error() + case cur.FP2 != "": + return cur.FP2, "" + case cur.None2 != "": + return "", cur.None2 + } + return "", "its record carries no fingerprint of the new form" + } +} + func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }