Keep manifests and say why no new-form fingerprint exists, so the group check can start from facts (issues 1449, 1450, 1444, 1451, design 47 R4c-2)

modules/mesh-delivery-indexer/internal/index/candidates.go

@@ -43,8 +43,10 @@ type Indexer struct {
 	Store Store // candidates
 	// Modules is the modules bucket; nil writes no fingerprints.
 	Modules Store
-	Builds  *Builds
-	Logf    func(format string, args ...any)
+	// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
+	Manifests Store
+	Builds    *Builds
+	Logf      func(format string, args ...any)
 
 	mu     sync.Mutex // guards source, which the source watch and the forge messages both reach
 	source sourceNow
@@ -72,6 +74,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
 			return err
 		}
 	}
+	var digests map[string]string
+	if f.Kind == delivery.ForgePull {
+		var err error
+		if digests, err = ix.keepManifests(ctx, f); err != nil {
+			return err
+		}
+	}
 	if trunkLike[f.Branch] {
 		ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
 		return nil
@@ -89,7 +98,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
 			}
 			cur = &c
 		}
-		next := ix.next(cur, f)
+		next := ix.next(cur, f, digests)
+		if next != nil {
+			if err := ix.refuseDisagreement(*next); err != nil {
+				return nil, false, err
+			}
+		}
 		switch {
 		case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
 			return nil, false, nil
@@ -185,7 +199,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
 }
 
 // next is the group after the message; nil when the group is left with no pull request.
-func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate {
+//
+// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
+// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
+// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
+// directory.
+func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate {
 	var pulls []delivery.CandidatePull
 	var base uint64
 	if cur != nil {
@@ -202,13 +221,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
 		switch {
 		case f.DirsKnown:
 			for _, dir := range f.Dirs {
-				if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) {
+				m := ix.Builds.ModuleOf(f.Repository, dir)
+				if !slices.Contains(entry.Modules, m) {
 					entry.Modules = append(entry.Modules, m)
 				}
+				if digest, ok := digests[dir]; ok {
+					if entry.Fps == nil {
+						entry.Fps = map[string]delivery.CandidateModule{}
+					}
+					entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
+				}
 			}
 		case at >= 0 && pulls[at].Head.Head == f.Head:
 			// Its directories are not known now, for the head the group already holds: what it knew stands.
-			entry.Modules = pulls[at].Modules
+			entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
 		}
 		if at >= 0 {
 			pulls[at] = entry
@@ -230,5 +256,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
 		heads[i] = p.Head
 	}
 	hash, _ := delivery.HeadsHash(heads)
-	return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls}
+	return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
+}
+
+// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
+// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
+const NoneNoPreview = "no merge preview is computed"
+
+// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
+// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
+// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
+// request is judged again.
+func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
+	first := map[string]delivery.CandidatePull{}
+	for _, p := range c.Pulls {
+		for module, m := range p.Fps {
+			before, ok := first[module]
+			if !ok {
+				first[module] = p
+				continue
+			}
+			if before.Fps[module] != m {
+				err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
+					ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
+				return err
+			}
+		}
+	}
+	return nil
 }

modules/mesh-delivery-indexer/internal/index/inputs.go

@@ -0,0 +1,115 @@
+package index
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq
+// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport.
+const (
+	// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
+	None2NoManifest = "the message carries no manifest"
+	// None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing.
+	None2Builder = "builder's fingerprint not yet known"
+	// None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none
+	// either, NoneContext).
+	None2Context = "context not pinned"
+)
+
+// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's
+// directory (the bridge reads these, internal/bridge LockFiles; issue 1450).
+var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"}
+
+// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450).
+type moduleInputs struct {
+	// Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none.
+	Kept *delivery.KeptInputs2
+	// Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by
+	// artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the
+	// watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then.
+	Languages map[string]string
+	// None2 is why the module has no fingerprint of the new form.
+	None2 string
+}
+
+// baseFP answers a base module's new-form fingerprint, or why it has none.
+type baseFP func(module string) (fp, none string)
+
+// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock
+// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's
+// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why.
+//
+// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a
+// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
+// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its
+// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive,
+// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module.
+func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
+	m, err := catalogue.ParseManifest([]byte(raw))
+	if err != nil {
+		return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
+	}
+	if m.Module != module {
+		return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
+	}
+	if tree == "" {
+		return moduleInputs{None2: "the merge names no tree for the module's directory"}
+	}
+	kept := delivery.KeptInputs2{Tree: tree}
+	langs := map[string]string{}
+	contexts := false
+	var missing []string
+	if m.Build != nil {
+		for _, a := range m.Build.Artifacts {
+			if a.Context != nil {
+				contexts = true
+			}
+			if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage {
+				continue
+			}
+			name, ok := lockFile[a.Language]
+			if !ok {
+				continue
+			}
+			langs[a.Name] = a.Language
+			if kept.Locks == nil {
+				kept.Locks = map[string]string{}
+			}
+			kept.Locks[a.Name] = locks[name]
+		}
+		seen := map[string]bool{}
+		for _, on := range m.Build.On {
+			if on.Module == "" || seen[on.Module] {
+				continue
+			}
+			seen[on.Module] = true
+			if fp, none := base(on.Module); fp != "" {
+				kept.Bases = append(kept.Bases, fp)
+			} else {
+				missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none))
+			}
+		}
+	}
+	sort.Strings(kept.Bases)
+	sort.Strings(missing)
+	out := moduleInputs{Kept: &kept}
+	if len(langs) > 0 {
+		out.Languages = langs
+	}
+	switch {
+	case contexts:
+		out.None2 = None2Context
+	case len(missing) > 0:
+		out.None2 = strings.Join(missing, "; ")
+	case len(langs) > 0:
+		out.None2 = delivery.ToolchainNoReport
+	default:
+		out.None2 = None2Builder
+	}
+	return out
+}

modules/mesh-delivery-indexer/internal/index/manifests.go

@@ -0,0 +1,83 @@
+package index
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests
+// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the
+// module a build record names the directory the path of.
+func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string {
+	var m struct {
+		Module string `json:"module"`
+	}
+	if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" {
+		return m.Module
+	}
+	name, _ := ix.Builds.Module(repository, dir)
+	return name
+}
+
+// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
+// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for
+// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the
+// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried
+// again. With no manifests store nothing is written and the digest is still answered.
+func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) {
+	digest := delivery.ManifestDigest([]byte(raw))
+	if ix.Manifests == nil {
+		return digest, nil
+	}
+	key, err := delivery.ManifestsKey(digest)
+	if err != nil {
+		return "", err
+	}
+	data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
+	if err != nil {
+		return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err)
+	}
+	err = ix.Manifests.Create(ctx, key, data)
+	if err == nil {
+		return digest, nil
+	}
+	if !errors.Is(err, ErrConflict) {
+		return "", fmt.Errorf("writing manifests.%s: %w", key, err)
+	}
+	held, _, found, err := ix.Manifests.Get(ctx, key)
+	if err != nil {
+		return "", fmt.Errorf("reading manifests.%s: %w", key, err)
+	}
+	if !found {
+		return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key)
+	}
+	if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) {
+		return digest, nil
+	}
+	return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module)
+}
+
+// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
+// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
+func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
+	digests := make(map[string]string, len(f.Manifests))
+	for dir, raw := range f.Manifests {
+		module := ix.moduleNameOf(f.Repository, dir, raw)
+		digest, err := ix.keepManifest(ctx, module, raw)
+		switch {
+		case err == nil:
+			digests[dir] = digest
+		case errors.Is(err, errCannotBeAValue):
+			ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err)
+		default:
+			return nil, err
+		}
+	}
+	return digests, nil
+}
+
+var errCannotBeAValue = errors.New("not a manifest value")

modules/mesh-delivery-indexer/internal/index/modules.go

@@ -3,6 +3,7 @@ package index
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"sort"
 	"strings"
 
@@ -47,8 +48,14 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
 // applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
 // why), with the source revision that names the merge commit as the repository's head when the indexer has seen it
 // (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
-// holds is not written again.
+// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
+// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
+// 1450); this release writes no fingerprint of the new form for any module.
 func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
+	digests, err := ix.keepManifests(ctx, f)
+	if err != nil {
+		return err
+	}
 	for _, k := range ix.Builds.Modules(f.Repository) {
 		key, err := delivery.ModulesKey(k.Module)
 		if err != nil {
@@ -56,10 +63,22 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
 			continue
 		}
 		fp, none := fingerprintOf(k, f.Trees)
-		next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge)}
+		next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path]}
+		// The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which
+		// names what is missing, from its manifest and lock hashes (inputs.go).
+		raw, has := f.Manifests[k.Path]
+		switch {
+		case !has:
+			next.None2 = None2NoManifest
+		case next.Manifest == "":
+			next.None2 = "the manifest cannot be read: it cannot be kept as a value"
+		default:
+			in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx))
+			next.Inputs2, next.None2 = in.Kept, in.None2
+		}
 		err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
 			if found {
-				if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) {
+				if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
 					return nil, false, nil
 				}
 			}
@@ -73,4 +92,31 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
 	return nil
 }
 
+// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
+func (ix *Indexer) baseFP(ctx context.Context) baseFP {
+	return func(module string) (string, string) {
+		key, err := delivery.ModulesKey(module)
+		if err != nil {
+			return "", err.Error()
+		}
+		value, _, found, err := ix.Modules.Get(ctx, key)
+		if err != nil {
+			return "", "its record cannot be read: " + err.Error()
+		}
+		if !found {
+			return "", "the indexer holds no record of it"
+		}
+		cur, err := delivery.DecodeModule(value)
+		switch {
+		case err != nil:
+			return "", "its record cannot be read: " + err.Error()
+		case cur.FP2 != "":
+			return cur.FP2, ""
+		case cur.None2 != "":
+			return "", cur.None2
+		}
+		return "", "its record carries no fingerprint of the new form"
+	}
+}
+
 func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }