diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b0d9a8a..0e56eac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -293,14 +293,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer calls.Close() // The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears. verdicts := process.AwaitBucket(ctx, js, conn, index.VerdictsBucket, "no verdict on a manifest is written", logger.Printf) + // The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every + // fingerprint takes the version line. + packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf) ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), - ReadOpen: openVia(calls.Call)} + ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call)} // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). var started atomic.Bool + var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error { if err := known.Put(le.Value); err != nil { logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err) @@ -310,6 +314,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil { logger.Printf("the registered module %s is not indexed: %v", le.Key, err) } + // A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are. + if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil { + logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err) + } + } else { + startEnds = append(startEnds, le) } return nil }); err != nil { @@ -364,6 +374,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as // having no toolchain reported, and back). + // The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and + // the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4). + packages.Ready(func(jetstream.KeyValue) { + if err := ix.FillPackages(ctx); err != nil { + logger.Printf("publishing the packages at start: %v", err) + } + }) + for _, le := range startEnds { + if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil { + logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err) + } + } + startEnds = nil if err := ix.RecomputeAtStart(ctx); err != nil { logger.Printf("computing the new-form fingerprints at start: %v", err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 394af11..e3f31b2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-indexer/internal/fingerprint" "mesh-delivery-indexer/internal/index" ) @@ -78,6 +79,65 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) } } +// filesVia asks the forge for every blob under each directory at a commit (the same tool with `files`, ADR 0593 §2): it +// answers {files} by directory, then path relative to it, each with its blob's object id. An answer that lists no +// directory asked is an error naming it; a cut listing fails the call in the tool. +func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { + return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return nil, fmt.Errorf("%q is not /", repository) + } + raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "files": dirs}) + if err != nil { + return nil, err + } + var a struct { + Files map[string]map[string]string `json:"files"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {files}: %w", commitTreesTool, err) + } + for _, d := range dirs { + if _, ok := a.Files[d]; !ok { + return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d) + } + } + return a.Files, nil + } +} + +// goSumVia reads a module directory's go.sum at a commit (gitea_get_file): the version line's packages of a module +// whose build reported no imports (ADR 0593 §2). A directory with no go.sum answers nothing. +func goSumVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) ([]byte, error) { + return func(_ context.Context, repository, sha, dir string) ([]byte, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return nil, fmt.Errorf("%q is not /", repository) + } + p := "go.sum" + if dir != "" && dir != delivery.RootPath { + p = dir + "/go.sum" + } + raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha}) + if err != nil { + if strings.Contains(err.Error(), ": 404 ") { + return nil, nil + } + return nil, err + } + var f struct { + File *struct { + Content *string `json:"content"` + } `json:"file"` + } + if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil { + return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err) + } + return []byte(*f.File.Content), nil + } +} + // containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor). func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) { return func(_ context.Context, repository, commit, in string) (bool, error) { @@ -166,7 +226,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str } out.Locks[dir][name] = hex.EncodeToString(sum[:]) if name == "go.sum" { - if p := delivery.MeshPackagesOf([]byte(c)); len(p) > 0 { + if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 { if out.Packages == nil { out.Packages = map[string][]string{} } diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go new file mode 100644 index 0000000..0935d7d --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -0,0 +1,362 @@ +// Package fingerprint is the one computation of a module's source fingerprint, of a named tree, of a mesh package's +// content digest and of a package's published version (novox/hq ADR 0593 §2): the indexer's own, and nobody else's. It +// left mesh-sdk's delivery package, which keeps only the fingerprint's form (delivery.SourcePrefix2, the key part) and +// the shapes the indexer writes (delivery.Module, delivery.PublishedPackages). No other part computes any of it: the +// builder, the checker and the build modules read what the indexer writes. +package fingerprint + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "maps" + "path" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// NoPackageVersion opens the reason a module has no fingerprint while a mesh package it uses has no published version +// the indexer holds (novox/hq ADR 0573 §2); the packages follow, by module path. +const NoPackageVersion = "no published version of the package " + +// Inputs are what a module's source fingerprint is of (novox/hq ADR 0429 clause 5 as ADR 0437 clause 5 changed it; ADR +// 0453 clause 6; ADR 0593 §2). A base enters by its module's fingerprint, an artifact that resolves packages when it is +// built by the hash of its lock file (an entry with an empty hash is such an artifact with no lock, which has no +// fingerprint), an artifact's toolchain as delivery.Toolchain2. +// +// The mesh packages enter in one of two forms. **The import form** (ADR 0593 §2): Imports, the content digest of each +// mesh package the module's programs import (by import path), and Requires, the digest of each mesh Go module's go.mod +// and go.sum (by module path). **The version line** (ADR 0573 §2), for a module with no reported import set: +// Packages, the published version of each mesh Go module its lock file names. A caller gives one form, never both. +type Inputs struct { + Module string + Tree string + Bases []string + Builder string + Contexts map[string]string + Locks map[string]string + Toolchains map[string]delivery.Toolchain2 + Packages map[string]string + Imports map[string]string + Requires map[string]string + Unpinned string +} + +// Of is the module's source fingerprint, `src2:` and the hex sha256 of its inputs' lines, or none and the reason when +// the source does not pin the build: Unpinned when set, "no tree" when Tree is empty, "no lock" when an artifact +// resolves packages and has no lock, NoPackageVersion when a package of the version line has none. It is a pure +// function of its inputs. The lines are "module ", "tree " and "builder ", then, sorted, "base " (once +// each), "context ", "lock ", "toolchain +// ", "package ", "import " and "requires ", joined by a +// newline. Inputs with no package of either form give the value they gave before packages entered. +func Of(in Inputs) (fp, none string) { + if in.Unpinned != "" { + return "", in.Unpinned + } + if in.Tree == "" { + return "", "no tree" + } + for _, h := range in.Locks { + if h == "" { + return "", "no lock" + } + } + var unpublished []string + for p, v := range in.Packages { + if v == "" { + unpublished = append(unpublished, p) + } + } + if len(unpublished) > 0 { + sort.Strings(unpublished) + return "", NoPackageVersion + strings.Join(unpublished, ", ") + } + lines := []string{"module " + in.Module, "tree " + in.Tree, "builder " + in.Builder} + bases := map[string]bool{} + for _, b := range in.Bases { + bases[b] = true + } + for b := range bases { + lines = append(lines, "base "+b) + } + for a, t := range in.Contexts { + lines = append(lines, "context "+a+" "+t) + } + for a, h := range in.Locks { + lines = append(lines, "lock "+a+" "+h) + } + for a, t := range in.Toolchains { + lines = append(lines, "toolchain "+a+" "+t.Language+" "+t.Version+" "+t.Recipe) + } + for p, v := range in.Packages { + lines = append(lines, "package "+p+" "+v) + } + for p, d := range in.Imports { + lines = append(lines, "import "+p+" "+d) + } + for m, d := range in.Requires { + lines = append(lines, "requires "+m+" "+d) + } + sort.Strings(lines[3:]) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return delivery.SourcePrefix2 + hex.EncodeToString(sum[:]), "" +} + +// NamedTree is the tree input of a module whose every artifact names the paths it is built from (novox/hq ADR 0499 +// clause 2): `named:` and the hex sha256 of the lines "manifest " and, sorted, "path ", joined by +// a newline. manifest is the module's delivery.ManifestDigest; paths are the named paths, each clean and relative to +// the module's directory, with the git object id each has at the commit. It refuses no path, a named path with no +// object, a path that is not clean, and a manifest that is not a digest, naming them: the module then has none. +func NamedTree(manifest string, paths map[string]string) (string, error) { + if _, err := delivery.ManifestsKey(manifest); err != nil { + return "", err + } + if len(paths) == 0 { + return "", fmt.Errorf("the module names no path, so it has no named tree") + } + lines := make([]string, 0, len(paths)) + for p, object := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return "", fmt.Errorf("the named path %q is not a clean path inside the module's directory", p) + } + if object == "" { + return "", fmt.Errorf("the named path %q has no git object at the commit", p) + } + lines = append(lines, "path "+p+" "+object) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n"))) + return delivery.NamedTreePrefix + hex.EncodeToString(sum[:]), nil +} + +// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under +// delivery.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set +// takes the version line of (ADR 0593 §2). +func MeshPackagesOf(goSum []byte) []string { + seen := map[string]bool{} + for _, line := range strings.Split(string(goSum), "\n") { + f := strings.Fields(line) + if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) { + continue + } + if _, _, ok := delivery.PackageRepository(f[0]); ok { + seen[f[0]] = true + } + } + if len(seen) == 0 { + return nil + } + return slices.Sorted(maps.Keys(seen)) +} + +// OtherPackages are the packages not published from repository, the module's own, in their order; nil for none. +func OtherPackages(packages []string, repository string) []string { + var out []string + for _, p := range packages { + if repo, _, ok := delivery.PackageRepository(p); ok && !delivery.SameRepository(repo, repository) { + out = append(out, p) + } + } + return out +} + +// Resolved is an import set resolved against the published packages: the digest of each import, the requirements' +// digest of each mesh Go module holding one, the version to build each of another repository's against, and the +// imports none publishes. +type Resolved struct { + Digests map[string]string + Requires map[string]string + Versions delivery.PackageVersions + Missing []string +} + +// Resolve resolves the imports of a module of repository against the published packages (by module path): each +// import is held by the published module with the longest path that publishes it. A module of repository itself is +// built from its directory, so it is given no version (a same-repository replace, ADR 0593 §1); it still enters by +// its digests. +func Resolve(repository string, imports []string, published map[string]delivery.PublishedPackages) Resolved { + r := Resolved{Digests: map[string]string{}, Requires: map[string]string{}} + for _, imp := range imports { + var holder *delivery.PublishedPackages + digest := "" + for _, m := range slices.Sorted(maps.Keys(published)) { + p := published[m] + if imp != m && !strings.HasPrefix(imp, m+"/") { + continue + } + d, ok := p.DigestOf(imp) + if !ok { + continue + } + if holder == nil || len(m) > len(holder.Module) { + holder, digest = &p, d + } + } + if holder == nil { + r.Missing = append(r.Missing, imp) + continue + } + r.Digests[imp] = digest + r.Requires[holder.Module] = holder.Requires + if !delivery.SameRepository(holder.Repository, repository) { + if r.Versions == nil { + r.Versions = delivery.PackageVersions{} + } + r.Versions[holder.Module] = holder.Commit + } + } + slices.Sort(r.Missing) + return r +} + +// Packages are a mesh Go module's requirements' digest and the content digest of each of its packages, from files, +// every blob under the module's directory by path relative to it with its git object id (the forge's listing, +// gitea_commit_trees with files). It refuses a listing with no go.mod at the module's root. +// +// A package is a directory that directly holds a .go file that is not a test (`testdata`, and a directory whose name +// starts with `_` or `.`, is none, and nor is anything under a directory holding a go.mod of its own, another +// module). Its digest is the hex sha256 of the sorted lines " " of every file under it, +// leaving out `*_test.go` and everything under a sub-directory that is itself a package or holds a go.mod: a file +// //go:embed can reach counts, a test or a sub-package does not. The requirements' digest is the hex sha256 of the +// lines "go.mod " and "go.sum " (the second absent with no go.sum). +func Packages(module string, files map[string]string) (requires string, packages []delivery.PackageDigest, err error) { + goMod, ok := files["go.mod"] + if !ok { + return "", nil, fmt.Errorf("the module %s has no go.mod at its directory", module) + } + req := "go.mod " + goMod + if goSum, ok := files["go.sum"]; ok { + req += "\ngo.sum " + goSum + } + sum := sha256.Sum256([]byte(req)) + requires = hex.EncodeToString(sum[:]) + + nested := map[string]bool{} // directories holding a go.mod of their own + for p := range files { + if dir, name := path.Split(p); name == "go.mod" && dir != "" { + nested[strings.TrimSuffix(dir, "/")] = true + } + } + inNested := func(dir string) bool { + for d := dir; d != "." && d != ""; d = path.Dir(d) { + if nested[d] { + return true + } + } + return false + } + isPackageDir := map[string]bool{} + for p := range files { + dir := path.Dir(p) + if dir == "." { + dir = "" + } + if !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || ignoredDir(dir) || inNested(dir) { + continue + } + isPackageDir[dir] = true + } + for _, dir := range slices.Sorted(maps.Keys(isPackageDir)) { + var lines []string + for p, object := range files { + rel, ok := under(p, dir) + if !ok || strings.HasSuffix(p, "_test.go") { + continue + } + if sub := path.Dir(rel); sub != "." && cutBySub(dir, sub, isPackageDir, nested) { + continue + } + lines = append(lines, rel+" "+object) + } + sort.Strings(lines) + s := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + imp := module + if dir != "" { + imp = module + "/" + dir + } + packages = append(packages, delivery.PackageDigest{Package: imp, Digest: hex.EncodeToString(s[:])}) + } + slices.SortFunc(packages, func(a, b delivery.PackageDigest) int { return strings.Compare(a.Package, b.Package) }) + return requires, packages, nil +} + +// ignoredDir is whether the go tool takes no package from dir: an element named testdata, or starting with _ or . +func ignoredDir(dir string) bool { + for _, e := range strings.Split(dir, "/") { + if e == "testdata" || strings.HasPrefix(e, "_") || strings.HasPrefix(e, ".") { + return true + } + } + return false +} + +// under is p relative to dir ("" for the module's root), and whether p is under it. +func under(p, dir string) (string, bool) { + if dir == "" { + return p, true + } + return strings.CutPrefix(p, dir+"/") +} + +// cutBySub is whether a file in the sub-directory sub of the package dir lies under another package or module. +func cutBySub(dir, sub string, isPackageDir, nested map[string]bool) bool { + acc := dir + for _, e := range strings.Split(sub, "/") { + if acc == "" { + acc = e + } else { + acc += "/" + e + } + if isPackageDir[acc] || nested[acc] { + return true + } + } + return false +} + +func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys(m)) } + +// HeldVersionsOf is the version line's version of each mesh Go module (by module path) as the indexer has always read +// it (novox/hq ADR 0573 §2), so a module with no reported import set keeps the fingerprint it had: the trunk merge +// commit of the held module of the package's repository whose directory holds the package's directory (the deepest, a +// module at the repository's root holding every directory). A package no held module holds, or whose module names no +// commit or no directory, is in missing, sorted; versions is nil when packages is empty. +func HeldVersionsOf(packages []string, modules map[string]delivery.Module) (versions delivery.PackageVersions, missing []string) { + for _, pkg := range packages { + repository, dir, ok := delivery.PackageRepository(pkg) + commit, depth := "", -1 + if ok { + for _, key := range slices.Sorted(maps.Keys(modules)) { + m := modules[key] + if !delivery.SameRepository(m.Repository, repository) || m.Commit == "" { + continue + } + at := strings.Trim(m.Path, "/") + switch { + case at == "": + continue // a value that names no directory says nothing of where the package lies + case at == delivery.RootPath: + at = "" + case dir != at && !strings.HasPrefix(dir, at+"/"): + continue + } + if d := len(at); d > depth { + commit, depth = m.Commit, d + } + } + } + if commit == "" { + missing = append(missing, pkg) + continue + } + if versions == nil { + versions = delivery.PackageVersions{} + } + versions[pkg] = commit + } + slices.Sort(missing) + return versions, missing +} diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go new file mode 100644 index 0000000..c63ab1b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go @@ -0,0 +1,161 @@ +package fingerprint + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const sdk = "git.novox.be/novox/mesh-sdk/go" + +func base() Inputs { + return Inputs{Module: "netcheck", Tree: "t1", Bases: []string{"src2:b"}, Contexts: map[string]string{"c": "x"}, + Locks: map[string]string{"bin": "l1"}, Toolchains: map[string]delivery.Toolchain2{"bin": {Language: "go", Version: "1.26.0", Recipe: "7"}}} +} + +// The form moved out of mesh-sdk computes what mesh-sdk's SourceFingerprint2 and NamedTree computed for the same +// inputs (golden values taken from mesh-sdk 3d87ddc), so the move itself rebuilds nothing. +func TestTheMovedComputationGivesTheValuesItGaveInMeshSDK(t *testing.T) { + in := base() + in.Packages = map[string]string{sdk: "abc"} + if fp, none := Of(in); fp != "src2:0f96be10d19712c94ea0ece8125a45cbb658ec06b705b28e747332cc94500e41" || none != "" { + t.Fatalf("with a package version: %q %q", fp, none) + } + if fp, _ := Of(base()); fp != "src2:f1a869faf14a3e422feba672156b42b1a7af7e7a1097c11554299f9e37e58625" { + t.Fatalf("without: %q", fp) + } + named, err := NamedTree(strings.Repeat("0", 64), map[string]string{"cmd": "t9", "go.mod": "b1"}) + if err != nil || named != "named:0290bf64fc93b1c10954797d548be30d936008b041c58261faad0c388a3d4c80" { + t.Fatalf("named tree: %q %v", named, err) + } + in.Packages = map[string]string{sdk: ""} + if fp, none := Of(in); fp != "" || !strings.HasPrefix(none, NoPackageVersion) { + t.Fatalf("a package with no version: %q %q", fp, none) + } +} + +func published(module, repo, dir, commit, requires string, digests map[string]string) delivery.PublishedPackages { + p := delivery.PublishedPackages{Module: module, Repository: repo, Dir: dir, Commit: commit, Requires: requires} + for _, pkg := range sortedKeys(digests) { + p.Packages = append(p.Packages, delivery.PackageDigest{Package: pkg, Digest: digests[pkg]}) + } + return p +} + +// ADR 0593 §2, §3: the fingerprint takes the digest of each package a module imports, and of each mesh Go module's +// requirements; a package it does not import moves nothing, an imported one or a sibling one moves it. +func TestTheImportFormMovesOnlyWithWhatTheModuleImports(t *testing.T) { + const sibling = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + imports := []string{sdk, sdk + "/announce", sibling} + pubs := func(announce, buildstep, sib, requires string) map[string]delivery.PublishedPackages { + return map[string]delivery.PublishedPackages{ + sdk: published(sdk, "novox/mesh-sdk", "go", "c1", requires, map[string]string{sdk: "r", sdk + "/announce": announce, sdk + "/buildstep": buildstep}), + sibling: published(sibling, "novox/mesh-catalog", "modules/mesh-build-go/buildstep", "c9", "q", map[string]string{sibling: sib}), + } + } + fpOf := func(p map[string]delivery.PublishedPackages) string { + t.Helper() + r := Resolve("novox/mesh-catalog", imports, p) + if len(r.Missing) > 0 { + t.Fatalf("missing %v", r.Missing) + } + in := base() + in.Imports, in.Requires = r.Digests, r.Requires + fp, none := Of(in) + if fp == "" { + t.Fatal(none) + } + return fp + } + first := fpOf(pubs("a1", "b1", "s1", "q1")) + if got := fpOf(pubs("a1", "b2", "s1", "q1")); got != first { + t.Fatal("a package the module does not import (buildstep) moved its fingerprint") + } + if got := fpOf(pubs("a2", "b1", "s1", "q1")); got == first { + t.Fatal("an imported package (announce) changed and the fingerprint did not move") + } + if got := fpOf(pubs("a1", "b1", "s2", "q1")); got == first { + t.Fatal("a sibling package of the module's own repository changed and the fingerprint did not move") + } + if got := fpOf(pubs("a1", "b1", "s1", "q2")); got == first { + t.Fatal("mesh-sdk's requirements changed and the fingerprint did not move") + } + // Versions are pinned only for modules of another repository: a sibling is built from its directory. + r := Resolve("novox/mesh-catalog", imports, pubs("a1", "b1", "s1", "q1")) + if len(r.Versions) != 1 || r.Versions[sdk] != "c1" { + t.Fatalf("versions %v", r.Versions) + } + if r := Resolve("novox/mesh-catalog", append(imports, sdk+"/gone"), pubs("a1", "b1", "s1", "q1")); len(r.Missing) != 1 { + t.Fatalf("an import nobody publishes is missing: %v", r.Missing) + } +} + +// A package's digest is its own files: a test file, a sub-package or a nested module changes nothing of it; a file +// of it, or of a sub-directory that is no package (testdata), changes only it. +func TestAPackageDigestIsItsOwnFiles(t *testing.T) { + files := map[string]string{ + "go.mod": "m1", "go.sum": "s1", "stdio.go": "a", "stdio_test.go": "t", + "announce/announce.go": "b", "announce/testdata/x.json": "d", + "buildstep/step.go": "c", "buildstep/conformance/c.go": "e", + "tools/go.mod": "nm", "tools/t.go": "f", "internal/x/x.go": "g", "_old/o.go": "h", + } + requires, pkgs, err := Packages(sdk, files) + if err != nil { + t.Fatal(err) + } + digests := map[string]string{} + for _, p := range pkgs { + digests[p.Package] = p.Digest + } + want := []string{sdk, sdk + "/announce", sdk + "/buildstep", sdk + "/buildstep/conformance", sdk + "/internal/x"} + if got := sortedKeys(digests); strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("packages %v, want %v", got, want) + } + change := func(path, object string) map[string]string { + _, ps, err := Packages(sdk, with(files, path, object)) + if err != nil { + t.Fatal(err) + } + out := map[string]string{} + for _, p := range ps { + out[p.Package] = p.Digest + } + return out + } + moved := func(after map[string]string) (out []string) { + for p, d := range after { + if digests[p] != d { + out = append(out, p) + } + } + return out + } + for path, want := range map[string]string{ + "stdio_test.go": "", + "buildstep/conformance/c.go": sdk + "/buildstep/conformance", + "announce/testdata/x.json": sdk + "/announce", + "stdio.go": sdk, + "tools/t.go": "", + } { + got := moved(change(path, "zz")) + if (want == "" && len(got) != 0) || (want != "" && (len(got) != 1 || got[0] != want)) { + t.Errorf("changing %s moved %v, want %q", path, got, want) + } + } + if r2, _, _ := Packages(sdk, with(files, "go.sum", "s2")); r2 == requires { + t.Error("a changed go.sum left the requirements' digest") + } + if _, _, err := Packages(sdk, map[string]string{"a.go": "x"}); err == nil { + t.Error("a listing with no go.mod was taken as a module") + } +} + +func with(m map[string]string, k, v string) map[string]string { + out := map[string]string{} + for a, b := range m { + out[a] = b + } + out[k] = v + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 4784980..0ff54ad 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -58,6 +58,16 @@ type Indexer struct { JudgeID string // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. SayRefused func(ctx context.Context, r delivery.ManifestRefused) + // Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none, + // and every fingerprint takes the version line. + Packages Store + // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each + // with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from. + // Nil publishes no package. + Files func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) + // GoSum reads a module directory's go.sum at a commit (gitea.gitea_get_file): the mesh Go modules a module with no + // reported import set takes the version line of. Nil reads none: such a module keeps what it held. + GoSum func(ctx context.Context, repository, sha, dir string) ([]byte, error) // Now is the clock of a taken value; nil is time.Now. Now func() time.Time Builds *Builds @@ -80,19 +90,20 @@ type Indexer struct { mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow - previews map[string]delivery.Preview // by previews key (previews.go) - toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - recompute sync.Mutex // one Recompute at a time - untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) - filling atomic.Bool // a FillTaken runs in the background (KickFill) - fillAgain atomic.Bool // a kick arrived while it ran - judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) + previews map[string]delivery.Preview // by previews key (previews.go) + toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + recompute sync.Mutex // one Recompute at a time + untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) + filling atomic.Bool // a FillTaken runs in the background (KickFill) + fillAgain atomic.Bool // a kick arrived while it ran + atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished) + judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go new file mode 100644 index 0000000..4289b43 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -0,0 +1,74 @@ +package index + +import ( + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/fingerprint" +) + +// form is a module's fingerprint, or why it has none, and the version of each mesh Go module of another repository +// its build is pinned to. +type form struct { + fp, none string + versions delivery.PackageVersions +} + +// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the +// form they call for (novox/hq ADR 0593 §2, §4): +// +// - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is +// published (the packages state): the content digest of each imported package and the requirements' digest of each +// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only). +// - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs +// the build that reported the imports was built from. While the import form computes to it, nothing the build was +// built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions. +// Otherwise the import form is written, and ImportForm moves with it. +// - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module its +// go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), so its value is the one it had. +// +// It sets kept.ImportForm when it writes the import form. +func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, logf func(string, ...any)) form { + in := inputsFrom(module, *kept, toolchains) + if len(kept.Imports) > 0 { + r := fingerprint.Resolve(repository, kept.Imports, published) + if len(r.Missing) == 0 { + in.Imports, in.Requires = r.Digests, r.Requires + fp, none := fingerprint.Of(in) + switch { + case fp == "": + return form{none: none} + case kept.ImportForm == fp && prev.FP2 != "": + return form{fp: prev.FP2, versions: prev.Packages} // the build that reported the imports stands + } + kept.ImportForm = fp + return form{fp: fp, versions: r.Versions} + } + if logf != nil { + logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", ")) + } + } + versions, unpublished := fingerprint.HeldVersionsOf(kept.Packages, held) + if len(unpublished) > 0 && logf != nil { + logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) + } + if len(kept.Packages) > 0 { + in.Packages = map[string]string{} + for _, p := range kept.Packages { + in.Packages[p] = versions[p] + } + } + fp, none := fingerprint.Of(in) + if fp == "" { + return form{none: none} + } + return form{fp: fp, versions: versions} +} + +// inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of +// either form yet. +func inputsFrom(module string, k delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2) fingerprint.Inputs { + return fingerprint.Inputs{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts, + Locks: k.Locks, Toolchains: toolchains} +} diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go new file mode 100644 index 0000000..f659f8e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -0,0 +1,134 @@ +package index + +import ( + "context" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/fingerprint" +) + +// ApplyBuildEnd keeps the imports a done build reports at its end (delivery.BuildLogEntry.Imports; novox/hq ADR 0593 +// §2) on the module it built, when the build's fingerprint is the one the module holds: the import set of the module's +// last trunk build. With them it keeps ImportForm, the import-form fingerprint of exactly what the build was built from: +// the module's kept inputs and the toolchains its fingerprint was computed with, and each imported package's digest at +// the version the build took it at (another repository's at the version the module's fingerprint pins, its own +// repository's at the commit built). A recompute that gives that value again keeps the fingerprint (§4); one that +// differs writes the import form. The fingerprint itself is not changed here. +// +// An entry that is no done build's end, reports no imports, or is of a fingerprint the module no longer holds is left. +// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that +// resolves them writes the import form, one rebuild too many, never a stale module. +func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { + e, err := delivery.DecodeBuildLogEntry(value) + if err != nil { + return err + } + if e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" { + return nil + } + key, err := delivery.ModulesKey(e.Module) + if err != nil { + return nil + } + ix.recompute.Lock() + defer ix.recompute.Unlock() + raw, _, found, err := ix.Modules.Get(ctx, key) + if err != nil || !found { + return err + } + cur, err := delivery.DecodeModule(raw) + if err != nil || cur.FP2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" { + return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now + } + if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" { + return nil + } + _, m, err := ix.manifestRaw(ctx, cur.Manifest) + if err != nil { + return fmt.Errorf("modules.%s: its manifest: %w", key, err) + } + toolchains := map[string]delivery.Toolchain2{} + for artifact, language := range languagesOf(m) { + used, ok := cur.Toolchains[language] + if !ok { + ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language) + return nil + } + toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} + } + kept := *cur.Inputs2 + kept.Imports, kept.ImportForm = slices.Clone(e.Imports), "" + if built, err := ix.publishedAsBuilt(ctx, cur, e); err != nil { + ix.logf("%s: the packages its build imported cannot be read as built (%v): its imports are kept, and its next recompute writes the import form", e.Module, err) + } else if r := fingerprint.Resolve(cur.Repository, e.Imports, built); len(r.Missing) > 0 { + ix.logf("%s: no published package holds %v: its imports are kept, and the recompute that resolves them writes the import form", e.Module, r.Missing) + } else { + in := inputsFrom(e.Module, kept, toolchains) + in.Imports, in.Requires = r.Digests, r.Requires + if fp, _ := fingerprint.Of(in); fp != "" { + kept.ImportForm = fp + } + } + return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + now, err := delivery.DecodeModule(value) + if err != nil || now.FP2 != e.FP || now.Manifest != cur.Manifest || now.Inputs2 == nil { + return nil, false, nil + } + in := *now.Inputs2 + in.Imports, in.ImportForm = kept.Imports, kept.ImportForm + now.Inputs2 = &in + data, err := delivery.EncodeModule(now) + return data, false, err + }) +} + +// publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took +// it: a module of the built module's own repository at the commit built, one of another repository at the version the +// module's fingerprint pins (cur.Packages), else its latest published version. A module whose published commit is the +// one built is taken from the packages state; any other is read from the forge's listing at that commit. +func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e delivery.BuildLogEntry) (map[string]delivery.PublishedPackages, error) { + published := ix.publishedAll(ctx) + if r := fingerprint.Resolve(cur.Repository, e.Imports, published); len(r.Missing) > 0 { + if err := ix.FillPackages(ctx); err != nil { + return nil, err + } + published = ix.publishedAll(ctx) + } + type at struct{ repository, commit string } + reread := map[at][]string{} + for _, imp := range e.Imports { + r := fingerprint.Resolve(cur.Repository, []string{imp}, published) + for mod := range r.Requires { + p := published[mod] + commit := p.Commit + switch { + case delivery.SameRepository(p.Repository, cur.Repository) && e.Commit != "": + commit = e.Commit + case cur.Packages[mod] != "": + commit = cur.Packages[mod] + } + if commit != p.Commit { + k := at{p.Repository, commit} + if !slices.Contains(reread[k], mod) { + reread[k] = append(reread[k], mod) + } + } + } + } + out := maps.Clone(published) + for k, mods := range reread { + got, err := ix.packagesAt(ctx, k.repository, k.commit, mods, nil) + if err != nil { + return nil, err + } + maps.Copy(out, got) + } + return out, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go new file mode 100644 index 0000000..9d0a3bc --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -0,0 +1,155 @@ +package index + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const meshSDK = "git.novox.be/novox/mesh-sdk/go" + +// Full commits of mesh-sdk and of the catalogue. +var ( + c1 = strings.Repeat("1", 40) + c2 = strings.Repeat("2", 40) + c3 = strings.Repeat("3", 40) + cat1 = strings.Repeat("a", 40) +) + +// sdkFiles is a listing of mesh-sdk's go directory (gitea_commit_trees with files): announce and buildstep at the +// given blobs. +func sdkFiles(announce, buildstep string) map[string]string { + return map[string]string{"go.mod": "m1", "go.sum": "s1", "stdio.go": "r1", "announce/announce.go": announce, "buildstep/step.go": buildstep} +} + +// forge answers Files from the listing of each commit of mesh-sdk. +type forge map[string]map[string]string + +func (f forge) files(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) { + out := map[string]map[string]string{} + for _, d := range dirs { + if repository != "novox/mesh-sdk" || d != "go" || f[sha] == nil { + return nil, &missingDir{d} + } + out[d] = f[sha] + } + return out, nil +} + +type missingDir struct{ dir string } + +func (m *missingDir) Error() string { return "no directory " + m.dir } + +// ADR 0593 §2-§4 (D1, D2, D3): a build's imports are kept with no fingerprint moving; a mesh-sdk merge that changes +// a package the module does not import leaves it; one that changes an imported package moves it to the import form; +// and the packages state holds one key with a digest per package. +func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T) { + ctx := context.Background() + ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + listings := forge{c1: sdkFiles("a1", "b1"), c2: sdkFiles("a1", "b2"), c3: sdkFiles("a2", "b2")} + ix.Packages, ix.Files = newMem(), listings.files + // avahi's go.sum names mesh-sdk (the version line), as every Go module's does; mesh-sdk's module is held at c1. + setPackages(t, mods, "avahi", []string{meshSDK}) + sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"}) + if err != nil { + t.Fatal(err) + } + mods.put("mesh-sdk", sdkModule) + held, _ := ix.heldModules(ctx) + if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil { + t.Fatal(err) + } + pub := ix.publishedAll(ctx)[meshSDK] + if pub.Commit != c1 || len(pub.Packages) != 3 || pub.Requires == "" { + t.Fatalf("packages.%s = %+v, want commit c1 and a digest per package (go, announce, buildstep)", meshSDK, pub) + } + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + before, _ := moduleOf(t, mods, "avahi") + if before.FP2 == "" { + t.Fatalf("no fingerprint to start from: %q", before.None2) + } + + // The build of that fingerprint ends, reporting its imports: kept, and the fingerprint does not move. + end := buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce") + if err := ix.ApplyBuildEnd(ctx, end); err != nil { + t.Fatal(err) + } + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + after, _ := moduleOf(t, mods, "avahi") + if after.FP2 != before.FP2 || strings.Join(after.Inputs2.Imports, ",") != meshSDK+","+meshSDK+"/announce" || after.Inputs2.ImportForm == "" { + t.Fatalf("after the build's imports: fp2 %q (was %q), imports %v, import form %q; want the same fingerprint and the imports kept", + after.FP2, before.FP2, after.Inputs2.Imports, after.Inputs2.ImportForm) + } + + // mesh-sdk merges a change of buildstep only: avahi does not import it. + held, _ = ix.heldModules(ctx) + if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false); err != nil { + t.Fatal(err) + } + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "avahi"); m.FP2 != before.FP2 { + t.Fatalf("a mesh-sdk merge changing only buildstep moved avahi's fingerprint to %q", m.FP2) + } + + // mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version. + held, _ = ix.heldModules(ctx) + if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil { + t.Fatal(err) + } + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + moved, _ := moduleOf(t, mods, "avahi") + if moved.FP2 == before.FP2 || moved.FP2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 { + t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3", + moved.FP2, before.FP2, moved.Inputs2.ImportForm, moved.Packages) + } +} + +// A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept. +func TestABuildOfAnOlderFingerprintKeepsNoImports(t *testing.T) { + ctx := context.Background() + ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + if err := ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK)); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || len(m.Inputs2.Imports) != 0 { + t.Fatalf("imports %v kept from a build of another fingerprint", m.Inputs2) + } +} + +func buildEnd(t *testing.T, module, fp string, imports ...string) []byte { + t.Helper() + raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: fp, + Repository: catalogueRepo, Directory: "modules/" + module, Commit: cat1, Ref: "main", Node: "n1", At: time.Now(), + Outcome: delivery.BuildDone, Imports: imports}) + if err != nil { + t.Fatal(err) + } + return raw +} + +// setPackages writes a module's version-line packages, as a merge reading its go.sum keeps them. +func setPackages(t *testing.T, mods *memStore, module string, packages []string) { + t.Helper() + m, _ := moduleOf(t, mods, module) + in := *m.Inputs2 + in.Packages = packages + m.Inputs2 = &in + raw, err := delivery.EncodeModule(m) + if err != nil { + t.Fatal(err) + } + mods.put(module, raw) +} diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index dabca71..60f9553 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-indexer/internal/fingerprint" "mesh-delivery-indexer/judge" ) @@ -55,7 +56,7 @@ type baseFP func(module string) (fp, none string) // language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages, // resolves packages when it is built (wantsPackages, :1017) only when its language's lock file exists: it then has // an entry in Locks with that file's hash, and an artifact whose lock file the merge did not carry has no entry (a -// dependency-free or vendored bundle resolves nothing, as mesh-sdk SourceInputs2 says of an artifact not in Locks; +// dependency-free or vendored bundle resolves nothing, as fingerprint.Inputs says of an artifact not in Locks; // novox/hq ADR 0469 clause 5, issue 1459). A bundle that needs a lock and lacks one is refused where it is built, not // here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve // nothing and have no toolchain. A base is a build.on entry that names a module. @@ -65,7 +66,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo // inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery // Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its -// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs +// manifest, by the indexer's one function (fingerprint.NamedTree), and keeps their objects, so a computation from kept inputs // gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it // none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given. // @@ -104,7 +105,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, p if len(missing) > 0 { return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths} } - if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil { + if tree, err = fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil { return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths} } } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index b786d1b..5fc9569 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -9,6 +9,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" + "mesh-delivery-indexer/internal/fingerprint" "mesh-delivery-indexer/judge" ) @@ -97,9 +98,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - packages := delivery.OtherPackages(f.Packages[dir], f.Repository) // its own repository's come with its tree + packages := ix.goSumPackages(ctx, f, dir, cur, has) in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx)) next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths + if in.Kept != nil && has && cur.Inputs2 != nil { + // The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4). + in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm + } if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) { // The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK // that reads it judges the module again at start with no merge (issue 1923). @@ -118,6 +123,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, return w, err } } + // The packages this merge publishes come first, so the modules importing a changed one move in this recompute (ADR + // 0593 §3). A forge that cannot list them leaves the state as it was, said: the importers move at the next merge. + reach := maps.Clone(held) + maps.Copy(reach, overlay) + if err := ix.publish(ctx, f.Repository, f.Merge, reach, false); err != nil { + ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err) + } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. return w, ix.recomputeWith(ctx, overlay, w) } @@ -320,3 +332,39 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string in.Packages = slices.Clone(packages) return in } + +// goSumPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names (the version +// line's packages, ADR 0573 §2), read by the indexer itself (the bridge reads none since ADR 0593 §2): those the message +// carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read from the +// forge (GoSum). A directory with no go.sum has none. +func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string { + if carried := f.Packages[dir]; len(carried) > 0 { + return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree + } + hash := f.Locks[dir]["go.sum"] + if hash == "" { + return nil + } + if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) { + for _, h := range cur.Inputs2.Locks { + if h == hash { + return cur.Inputs2.Packages + } + } + } + if ix.GoSum == nil { + if has && cur.Inputs2 != nil { + return cur.Inputs2.Packages + } + return nil + } + raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) + if err != nil { + ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) + if has && cur.Inputs2 != nil { + return cur.Inputs2.Packages + } + return nil + } + return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository) +} diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go new file mode 100644 index 0000000..de1f0e7 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -0,0 +1,326 @@ +package index + +import ( + "context" + "fmt" + "maps" + "path" + "reflect" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/fingerprint" +) + +// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest +// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and +// digests in its state `packages` (delivery.PackagesBucket), one key per Go module. The builder, the checker and the +// build modules read it there; nobody else computes a digest or a version. +// +// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build +// reported (KeptInputs2.Imports). Its key is written at each merge of its repository, before the merge's recompute, so +// a merge that changes a package moves its importers in the same recompute (§3); and once at the start, for each +// reached module the state does not hold yet (FillPackages). + +// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer. +func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.PublishedPackages { + out := map[string]delivery.PublishedPackages{} + if ix.Packages == nil { + return out + } + keys, err := ix.Packages.Keys(ctx) + if err != nil { + ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err) + return out + } + for _, key := range keys { + value, _, found, err := ix.Packages.Get(ctx, key) + if err != nil || !found { + continue + } + p, err := delivery.DecodePublishedPackages(value) + if err != nil { + ix.logf("packages.%s holds what is not a published module, left out: %v", key, err) + continue + } + out[p.Module] = p + } + return out +} + +// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds +// reported. +func reached(held map[string]delivery.Module) (modules, imports map[string][]string) { + modules, imports = map[string][]string{}, map[string][]string{} + add := func(to map[string][]string, p string) { + repo, _, ok := delivery.PackageRepository(p) + if ok && !slices.Contains(to[repo], p) { + to[repo] = append(to[repo], p) + } + } + for _, m := range held { + if m.Inputs2 == nil { + continue + } + for _, p := range m.Inputs2.Packages { + add(modules, p) + } + for _, p := range m.Inputs2.Imports { + add(imports, p) + } + } + for _, by := range []map[string][]string{modules, imports} { + for r := range by { + slices.Sort(by[r]) + } + } + return modules, imports +} + +// packagesAt are the Go modules of repository at commit that hold modules (module paths) or imports (import paths), +// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its +// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge +// directory by directory). A module the listing has no go.mod for is said and left out. +func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]delivery.PublishedPackages, error) { + if ix.Files == nil || commit == "" { + return nil, nil + } + dirOf := map[string]string{} // module path to its directory in the repository + for _, m := range modules { + if _, dir, ok := delivery.PackageRepository(m); ok { + dirOf[m] = dir + } + } + holds := func(imp string) bool { + for m := range dirOf { + if imp == m || strings.HasPrefix(imp, m+"/") { + return true + } + } + return false + } + listings := map[string]map[string]string{} // by directory + list := func(dirs []string) error { + var ask []string + for _, d := range dirs { + if _, done := listings[d]; !done { + ask = append(ask, d) + } + } + if len(ask) == 0 { + return nil + } + got, err := ix.Files(ctx, repository, commit, ask) + if err != nil { + return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err) + } + for _, d := range ask { + listings[d] = got[d] + } + return nil + } + for _, imp := range imports { + if holds(imp) { + continue + } + _, dir, _ := delivery.PackageRepository(imp) + for d := dir; ; d = parent(d) { + if err := list([]string{d}); err != nil { + return nil, err + } + if _, ok := listings[d]["go.mod"]; ok { + m := strings.TrimSuffix(delivery.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/") + dirOf[m] = d + break + } + if d == "" { + ix.logf("%s at %s: no go.mod holds the import %s, so it has no digest", repository, commit, imp) + break + } + } + } + if err := list(slices.Sorted(maps.Values(dirOf))); err != nil { + return nil, err + } + out := map[string]delivery.PublishedPackages{} + for _, m := range slices.Sorted(maps.Keys(dirOf)) { + dir := dirOf[m] + requires, pkgs, err := fingerprint.Packages(m, listings[dir]) + if err != nil { + ix.logf("%s at %s: %v; it is not published", repository, commit, err) + continue + } + d := dir + if d == "" { + d = delivery.RootPath + } + out[m] = delivery.PublishedPackages{Module: m, Repository: delivery.RepositoryName(repository), Dir: d, Commit: commit, + Requires: requires, Packages: pkgs} + } + return out, nil +} + +// parent is the directory above d, "" for the repository's root. +func parent(d string) string { + if d == "" { + return "" + } + p := path.Dir(d) + if p == "." { + return "" + } + return p +} + +// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach, +// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left. +func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]delivery.Module, onlyMissing bool) error { + if ix.Packages == nil || ix.Files == nil { + return nil + } + modules, imports := reached(held) + repo := delivery.RepositoryName(repository) + if len(modules[repo]) == 0 && len(imports[repo]) == 0 { + return nil + } + have := ix.publishedAll(ctx) + if onlyMissing { + // Imports a held module already resolves are not asked again. + var ms, is []string + for _, m := range modules[repo] { + if _, ok := have[m]; !ok { + ms = append(ms, m) + } + } + for _, imp := range imports[repo] { + if len(fingerprint.Resolve(repo, []string{imp}, have).Missing) > 0 { + is = append(is, imp) + } + } + if len(ms) == 0 && len(is) == 0 { + return nil + } + modules[repo], imports[repo] = ms, is + } + // An import a published module of the repository already holds is resolved by its module's directory. + for m, p := range have { + if delivery.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing { + modules[repo] = append(modules[repo], m) + } + } + got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo]) + if err != nil { + return err + } + for _, m := range slices.Sorted(maps.Keys(got)) { + next := got[m] + key, err := delivery.PackagesKey(m) + if err != nil { + ix.logf("%s: %v", m, err) + continue + } + err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if cur, err := delivery.DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit && + cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) { + return nil, false, nil + } + } + next.At = ix.now().UTC() + data, err := delivery.EncodePublishedPackages(next) + return data, false, err + }) + if err != nil { + return fmt.Errorf("packages.%s: %w", key, err) + } + } + return nil +} + +// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the +// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of. +// A repository with no taken merge is left for its next merge. +func (ix *Indexer) FillPackages(ctx context.Context) error { + if ix.Packages == nil || ix.Files == nil || ix.Modules == nil || ix.Taken == nil { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + modules, imports := reached(held) + repos := map[string]bool{} + for r := range modules { + repos[r] = true + } + for r := range imports { + repos[r] = true + } + for _, repo := range slices.Sorted(maps.Keys(repos)) { + key, err := delivery.RepositoryKey(repo) + if err != nil { + continue + } + value, _, found, err := ix.Taken.Get(ctx, key) + if err != nil || !found { + continue + } + t, err := delivery.DecodeTaken(value) + if err != nil { + continue + } + if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil { + ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err) + } + } + return nil +} + +// candidatePublished is the packages state as a candidate builds against it (novox/hq ADR 0573 §4): each Go module a +// member's repository publishes at the member's head (members, by repository), every other at its latest published +// version. A head's modules are read from the forge once and kept while the head is a member somewhere; one the forge +// cannot list is said and left at its published version. +func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]delivery.Module) map[string]delivery.PublishedPackages { + out := ix.publishedAll(ctx) + if len(members) == 0 { + return out + } + modules, imports := reached(held) + for _, repo := range slices.Sorted(maps.Keys(members)) { + head := members[repo] + if len(modules[repo]) == 0 && len(imports[repo]) == 0 { + continue + } + at := repo + "@" + head + ix.mu.Lock() + got, ok := ix.atHeads[at] + ix.mu.Unlock() + if !ok { + var ms []string + ms = append(ms, modules[repo]...) + for m, p := range out { + if delivery.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) { + ms = append(ms, m) + } + } + var err error + if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil { + ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err) + continue + } + ix.mu.Lock() + if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept { + ix.atHeads = map[string]map[string]delivery.PublishedPackages{} + } + ix.atHeads[at] = got + ix.mu.Unlock() + } + maps.Copy(out, got) + } + return out +} + +// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again. +const maxHeadsKept = 64 diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index b73d0a9..8bbb64a 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -9,6 +9,8 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/fingerprint" ) // The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull @@ -254,7 +256,15 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive out.None2 = NonePreviewNoManifest return out } - in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, delivery.OtherPackages(pm.Packages, pv.Repository), ix.baseFP(ctx)) + // The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer + // computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build + // module still names them. + trunk := held[m] + packages := fingerprint.OtherPackages(pm.Packages, pv.Repository) + if len(pm.Packages) == 0 && trunk.Inputs2 != nil { + packages = trunk.Inputs2.Packages + } + in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx)) if in.Kept == nil { out.None2 = in.None2 return out @@ -274,8 +284,29 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive toolchains[artifact] = t.tc } inputs := *in.Kept // no builder input (ADR 0530 §1) - versions := candidatePackages(inputs.Packages, held, members) - fp, none := delivery.SourceFingerprint2(inputs.WithPackages(m, toolchains, versions)) + var fp, none string + var versions delivery.PackageVersions + if trunk.Inputs2 != nil && len(trunk.Inputs2.Imports) > 0 { + inputs.Imports, inputs.ImportForm = trunk.Inputs2.Imports, trunk.Inputs2.ImportForm + } + // The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk + // computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed. + published := ix.candidatePublished(ctx, members, held) + if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 { + f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, nil) + fp, none, versions = f.fp, f.none, f.versions + } else { + inputs.Imports, inputs.ImportForm = nil, "" + versions = candidatePackages(inputs.Packages, held, members) + in := inputsFrom(m, inputs, toolchains) + if len(inputs.Packages) > 0 { + in.Packages = map[string]string{} + for _, p := range inputs.Packages { + in.Packages[p] = versions[p] + } + } + fp, none = fingerprint.Of(in) + } if fp == "" { out.None2 = none return out @@ -291,7 +322,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive // head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published // version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none. func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions { - versions, _ := delivery.PackageVersionsOf(packages, held) + versions, _ := fingerprint.HeldVersionsOf(packages, held) for _, p := range packages { repository, _, _ := delivery.PackageRepository(p) if head := members[repository]; head != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 0a4c1cf..1789b43 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -199,6 +199,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before} } tools := ix.agreed() + published := ix.publishedAll(ctx) type result struct { fp, none string tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint @@ -254,12 +255,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli } toolchains[artifact] = t.tc } - // The latest published version of each mesh package the module uses: the trunk merge of the module that holds it, - // as this recompute holds it, so a package's merge moves every dependent in the same recompute (ADR 0573 §2). - versions, unpublished := delivery.PackageVersionsOf(kept.Packages, held) - if len(unpublished) > 0 { - ix.logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) - } var r result switch { case k.contexts: @@ -268,13 +263,14 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, ok := stillComputed(module, kept, k, toolchains, versions); ok { - r = result{fp: fp, tcs: k.held.Toolchains, pkgs: versions} + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, published); ok { + r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: - r.fp, r.none = delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions)) + f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, published, ix.logf) + r.fp, r.none = f.fp, f.none if r.fp != "" { - r.tcs, r.pkgs = toolchainsUsed(toolchains), versions + r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions } } done[module] = r @@ -421,6 +417,7 @@ func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur deliv if in.Kept == nil { return cur, false } + in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm cur.Inputs2, cur.None2, cur.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths return cur, true } @@ -442,11 +439,12 @@ func hasContext(m catalogue.Manifest) bool { // (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's // fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were // computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs, -// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that -// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree. -func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, versions delivery.PackageVersions) (string, bool) { +// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any +// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the +// module has none until they agree. +func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages) (string, delivery.PackageVersions, bool) { if k.held.FP2 == "" { - return "", false + return "", nil, false } toolchains := map[string]delivery.Toolchain2{} for artifact, language := range k.languages { @@ -456,12 +454,12 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma } used, ok := k.held.Toolchains[language] if !ok { - return "", false + return "", nil, false } toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } - fp, none := delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions)) - return fp, none == "" && fp == k.held.FP2 + f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, nil) + return f.fp, f.versions, f.none == "" && f.fp == k.held.FP2 } // toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's). diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 48426fe..c03a970 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -11,6 +11,8 @@ import ( "slices" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, @@ -87,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want manifest-refused alone", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index cf4ae16..e8c3886 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -20,7 +20,8 @@ { "name": "config-candidates", "history": 5 }, { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 }, - { "name": "verdicts", "history": 5 } + { "name": "verdicts", "history": 5 }, + { "name": "packages", "history": 5 } ], "emits": [ "manifest-refused"