Indexer computes fingerprints alone, over the packages a module imports

The fingerprint, named tree and package versions move from mesh-sdk into the
indexer's internal/fingerprint (ADR 0593 §2). The indexer publishes each reached
mesh Go module's package digests from the forge's listing in its new packages
state, keeps the imports a done build reports, and writes the import form only
when an input differs from the build that reported them (§4), so the form change
costs no rebuild and a package merge moves only its importers (§3). Candidates
take the same form against the group's member heads (D1-D4).

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-02 18:25:57 +01:00
parent 511e4b7414
commit fba98ee80f
15 changed files with 1429 additions and 42 deletions
@@ -293,14 +293,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer calls.Close()
// The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears.
verdicts := process.AwaitBucket(ctx, js, conn, index.VerdictsBucket, "no verdict on a manifest is written", logger.Printf)
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call)}
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call)}
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
if err := known.Put(le.Value); err != nil {
logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err)
@@ -310,6 +314,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil {
logger.Printf("the registered module %s is not indexed: %v", le.Key, err)
}
// A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are.
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
}
} else {
startEnds = append(startEnds, le)
}
return nil
}); err != nil {
@@ -364,6 +374,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as
// having no toolchain reported, and back).
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
// the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4).
packages.Ready(func(jetstream.KeyValue) {
if err := ix.FillPackages(ctx); err != nil {
logger.Printf("publishing the packages at start: %v", err)
}
})
for _, le := range startEnds {
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
}
}
startEnds = nil
if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
@@ -11,6 +11,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/internal/index"
)
@@ -78,6 +79,65 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string)
}
}
// filesVia asks the forge for every blob under each directory at a commit (the same tool with `files`, ADR 0593 §2): it
// answers {files} by directory, then path relative to it, each with its blob's object id. An answer that lists no
// directory asked is an error naming it; a cut listing fails the call in the tool.
func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "files": dirs})
if err != nil {
return nil, err
}
var a struct {
Files map[string]map[string]string `json:"files"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s is not {files}: %w", commitTreesTool, err)
}
for _, d := range dirs {
if _, ok := a.Files[d]; !ok {
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
}
}
return a.Files, nil
}
}
// goSumVia reads a module directory's go.sum at a commit (gitea_get_file): the version line's packages of a module
// whose build reported no imports (ADR 0593 §2). A directory with no go.sum answers nothing.
func goSumVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
return func(_ context.Context, repository, sha, dir string) ([]byte, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
p := "go.sum"
if dir != "" && dir != delivery.RootPath {
p = dir + "/go.sum"
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
if err != nil {
if strings.Contains(err.Error(), ": 404 ") {
return nil, nil
}
return nil, err
}
var f struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return []byte(*f.File.Content), nil
}
}
// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor).
func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) {
return func(_ context.Context, repository, commit, in string) (bool, error) {
@@ -166,7 +226,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
}
out.Locks[dir][name] = hex.EncodeToString(sum[:])
if name == "go.sum" {
if p := delivery.MeshPackagesOf([]byte(c)); len(p) > 0 {
if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 {
if out.Packages == nil {
out.Packages = map[string][]string{}
}
@@ -0,0 +1,362 @@
// Package fingerprint is the one computation of a module's source fingerprint, of a named tree, of a mesh package's
// content digest and of a package's published version (novox/hq ADR 0593 §2): the indexer's own, and nobody else's. It
// left mesh-sdk's delivery package, which keeps only the fingerprint's form (delivery.SourcePrefix2, the key part) and
// the shapes the indexer writes (delivery.Module, delivery.PublishedPackages). No other part computes any of it: the
// builder, the checker and the build modules read what the indexer writes.
package fingerprint
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"maps"
"path"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// NoPackageVersion opens the reason a module has no fingerprint while a mesh package it uses has no published version
// the indexer holds (novox/hq ADR 0573 §2); the packages follow, by module path.
const NoPackageVersion = "no published version of the package "
// Inputs are what a module's source fingerprint is of (novox/hq ADR 0429 clause 5 as ADR 0437 clause 5 changed it; ADR
// 0453 clause 6; ADR 0593 §2). A base enters by its module's fingerprint, an artifact that resolves packages when it is
// built by the hash of its lock file (an entry with an empty hash is such an artifact with no lock, which has no
// fingerprint), an artifact's toolchain as delivery.Toolchain2.
//
// The mesh packages enter in one of two forms. **The import form** (ADR 0593 §2): Imports, the content digest of each
// mesh package the module's programs import (by import path), and Requires, the digest of each mesh Go module's go.mod
// and go.sum (by module path). **The version line** (ADR 0573 §2), for a module with no reported import set:
// Packages, the published version of each mesh Go module its lock file names. A caller gives one form, never both.
type Inputs struct {
Module string
Tree string
Bases []string
Builder string
Contexts map[string]string
Locks map[string]string
Toolchains map[string]delivery.Toolchain2
Packages map[string]string
Imports map[string]string
Requires map[string]string
Unpinned string
}
// Of is the module's source fingerprint, `src2:` and the hex sha256 of its inputs' lines, or none and the reason when
// the source does not pin the build: Unpinned when set, "no tree" when Tree is empty, "no lock" when an artifact
// resolves packages and has no lock, NoPackageVersion when a package of the version line has none. It is a pure
// function of its inputs. The lines are "module <m>", "tree <t>" and "builder <fp>", then, sorted, "base <fp>" (once
// each), "context <artifact> <tree>", "lock <artifact> <hash>", "toolchain <artifact> <language> <version>
// <recipe>", "package <module> <version>", "import <package> <digest>" and "requires <module> <digest>", joined by a
// newline. Inputs with no package of either form give the value they gave before packages entered.
func Of(in Inputs) (fp, none string) {
if in.Unpinned != "" {
return "", in.Unpinned
}
if in.Tree == "" {
return "", "no tree"
}
for _, h := range in.Locks {
if h == "" {
return "", "no lock"
}
}
var unpublished []string
for p, v := range in.Packages {
if v == "" {
unpublished = append(unpublished, p)
}
}
if len(unpublished) > 0 {
sort.Strings(unpublished)
return "", NoPackageVersion + strings.Join(unpublished, ", ")
}
lines := []string{"module " + in.Module, "tree " + in.Tree, "builder " + in.Builder}
bases := map[string]bool{}
for _, b := range in.Bases {
bases[b] = true
}
for b := range bases {
lines = append(lines, "base "+b)
}
for a, t := range in.Contexts {
lines = append(lines, "context "+a+" "+t)
}
for a, h := range in.Locks {
lines = append(lines, "lock "+a+" "+h)
}
for a, t := range in.Toolchains {
lines = append(lines, "toolchain "+a+" "+t.Language+" "+t.Version+" "+t.Recipe)
}
for p, v := range in.Packages {
lines = append(lines, "package "+p+" "+v)
}
for p, d := range in.Imports {
lines = append(lines, "import "+p+" "+d)
}
for m, d := range in.Requires {
lines = append(lines, "requires "+m+" "+d)
}
sort.Strings(lines[3:])
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return delivery.SourcePrefix2 + hex.EncodeToString(sum[:]), ""
}
// NamedTree is the tree input of a module whose every artifact names the paths it is built from (novox/hq ADR 0499
// clause 2): `named:` and the hex sha256 of the lines "manifest <digest>" and, sorted, "path <path> <object>", joined by
// a newline. manifest is the module's delivery.ManifestDigest; paths are the named paths, each clean and relative to
// the module's directory, with the git object id each has at the commit. It refuses no path, a named path with no
// object, a path that is not clean, and a manifest that is not a digest, naming them: the module then has none.
func NamedTree(manifest string, paths map[string]string) (string, error) {
if _, err := delivery.ManifestsKey(manifest); err != nil {
return "", err
}
if len(paths) == 0 {
return "", fmt.Errorf("the module names no path, so it has no named tree")
}
lines := make([]string, 0, len(paths))
for p, object := range paths {
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
return "", fmt.Errorf("the named path %q is not a clean path inside the module's directory", p)
}
if object == "" {
return "", fmt.Errorf("the named path %q has no git object at the commit", p)
}
lines = append(lines, "path "+p+" "+object)
}
sort.Strings(lines)
sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n")))
return delivery.NamedTreePrefix + hex.EncodeToString(sum[:]), nil
}
// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under
// delivery.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
// takes the version line of (ADR 0593 §2).
func MeshPackagesOf(goSum []byte) []string {
seen := map[string]bool{}
for _, line := range strings.Split(string(goSum), "\n") {
f := strings.Fields(line)
if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) {
continue
}
if _, _, ok := delivery.PackageRepository(f[0]); ok {
seen[f[0]] = true
}
}
if len(seen) == 0 {
return nil
}
return slices.Sorted(maps.Keys(seen))
}
// OtherPackages are the packages not published from repository, the module's own, in their order; nil for none.
func OtherPackages(packages []string, repository string) []string {
var out []string
for _, p := range packages {
if repo, _, ok := delivery.PackageRepository(p); ok && !delivery.SameRepository(repo, repository) {
out = append(out, p)
}
}
return out
}
// Resolved is an import set resolved against the published packages: the digest of each import, the requirements'
// digest of each mesh Go module holding one, the version to build each of another repository's against, and the
// imports none publishes.
type Resolved struct {
Digests map[string]string
Requires map[string]string
Versions delivery.PackageVersions
Missing []string
}
// Resolve resolves the imports of a module of repository against the published packages (by module path): each
// import is held by the published module with the longest path that publishes it. A module of repository itself is
// built from its directory, so it is given no version (a same-repository replace, ADR 0593 §1); it still enters by
// its digests.
func Resolve(repository string, imports []string, published map[string]delivery.PublishedPackages) Resolved {
r := Resolved{Digests: map[string]string{}, Requires: map[string]string{}}
for _, imp := range imports {
var holder *delivery.PublishedPackages
digest := ""
for _, m := range slices.Sorted(maps.Keys(published)) {
p := published[m]
if imp != m && !strings.HasPrefix(imp, m+"/") {
continue
}
d, ok := p.DigestOf(imp)
if !ok {
continue
}
if holder == nil || len(m) > len(holder.Module) {
holder, digest = &p, d
}
}
if holder == nil {
r.Missing = append(r.Missing, imp)
continue
}
r.Digests[imp] = digest
r.Requires[holder.Module] = holder.Requires
if !delivery.SameRepository(holder.Repository, repository) {
if r.Versions == nil {
r.Versions = delivery.PackageVersions{}
}
r.Versions[holder.Module] = holder.Commit
}
}
slices.Sort(r.Missing)
return r
}
// Packages are a mesh Go module's requirements' digest and the content digest of each of its packages, from files,
// every blob under the module's directory by path relative to it with its git object id (the forge's listing,
// gitea_commit_trees with files). It refuses a listing with no go.mod at the module's root.
//
// A package is a directory that directly holds a .go file that is not a test (`testdata`, and a directory whose name
// starts with `_` or `.`, is none, and nor is anything under a directory holding a go.mod of its own, another
// module). Its digest is the hex sha256 of the sorted lines "<path relative to it> <object>" of every file under it,
// leaving out `*_test.go` and everything under a sub-directory that is itself a package or holds a go.mod: a file
// //go:embed can reach counts, a test or a sub-package does not. The requirements' digest is the hex sha256 of the
// lines "go.mod <object>" and "go.sum <object>" (the second absent with no go.sum).
func Packages(module string, files map[string]string) (requires string, packages []delivery.PackageDigest, err error) {
goMod, ok := files["go.mod"]
if !ok {
return "", nil, fmt.Errorf("the module %s has no go.mod at its directory", module)
}
req := "go.mod " + goMod
if goSum, ok := files["go.sum"]; ok {
req += "\ngo.sum " + goSum
}
sum := sha256.Sum256([]byte(req))
requires = hex.EncodeToString(sum[:])
nested := map[string]bool{} // directories holding a go.mod of their own
for p := range files {
if dir, name := path.Split(p); name == "go.mod" && dir != "" {
nested[strings.TrimSuffix(dir, "/")] = true
}
}
inNested := func(dir string) bool {
for d := dir; d != "." && d != ""; d = path.Dir(d) {
if nested[d] {
return true
}
}
return false
}
isPackageDir := map[string]bool{}
for p := range files {
dir := path.Dir(p)
if dir == "." {
dir = ""
}
if !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || ignoredDir(dir) || inNested(dir) {
continue
}
isPackageDir[dir] = true
}
for _, dir := range slices.Sorted(maps.Keys(isPackageDir)) {
var lines []string
for p, object := range files {
rel, ok := under(p, dir)
if !ok || strings.HasSuffix(p, "_test.go") {
continue
}
if sub := path.Dir(rel); sub != "." && cutBySub(dir, sub, isPackageDir, nested) {
continue
}
lines = append(lines, rel+" "+object)
}
sort.Strings(lines)
s := sha256.Sum256([]byte(strings.Join(lines, "\n")))
imp := module
if dir != "" {
imp = module + "/" + dir
}
packages = append(packages, delivery.PackageDigest{Package: imp, Digest: hex.EncodeToString(s[:])})
}
slices.SortFunc(packages, func(a, b delivery.PackageDigest) int { return strings.Compare(a.Package, b.Package) })
return requires, packages, nil
}
// ignoredDir is whether the go tool takes no package from dir: an element named testdata, or starting with _ or .
func ignoredDir(dir string) bool {
for _, e := range strings.Split(dir, "/") {
if e == "testdata" || strings.HasPrefix(e, "_") || strings.HasPrefix(e, ".") {
return true
}
}
return false
}
// under is p relative to dir ("" for the module's root), and whether p is under it.
func under(p, dir string) (string, bool) {
if dir == "" {
return p, true
}
return strings.CutPrefix(p, dir+"/")
}
// cutBySub is whether a file in the sub-directory sub of the package dir lies under another package or module.
func cutBySub(dir, sub string, isPackageDir, nested map[string]bool) bool {
acc := dir
for _, e := range strings.Split(sub, "/") {
if acc == "" {
acc = e
} else {
acc += "/" + e
}
if isPackageDir[acc] || nested[acc] {
return true
}
}
return false
}
func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys(m)) }
// HeldVersionsOf is the version line's version of each mesh Go module (by module path) as the indexer has always read
// it (novox/hq ADR 0573 §2), so a module with no reported import set keeps the fingerprint it had: the trunk merge
// commit of the held module of the package's repository whose directory holds the package's directory (the deepest, a
// module at the repository's root holding every directory). A package no held module holds, or whose module names no
// commit or no directory, is in missing, sorted; versions is nil when packages is empty.
func HeldVersionsOf(packages []string, modules map[string]delivery.Module) (versions delivery.PackageVersions, missing []string) {
for _, pkg := range packages {
repository, dir, ok := delivery.PackageRepository(pkg)
commit, depth := "", -1
if ok {
for _, key := range slices.Sorted(maps.Keys(modules)) {
m := modules[key]
if !delivery.SameRepository(m.Repository, repository) || m.Commit == "" {
continue
}
at := strings.Trim(m.Path, "/")
switch {
case at == "":
continue // a value that names no directory says nothing of where the package lies
case at == delivery.RootPath:
at = ""
case dir != at && !strings.HasPrefix(dir, at+"/"):
continue
}
if d := len(at); d > depth {
commit, depth = m.Commit, d
}
}
}
if commit == "" {
missing = append(missing, pkg)
continue
}
if versions == nil {
versions = delivery.PackageVersions{}
}
versions[pkg] = commit
}
slices.Sort(missing)
return versions, missing
}
@@ -0,0 +1,161 @@
package fingerprint
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const sdk = "git.novox.be/novox/mesh-sdk/go"
func base() Inputs {
return Inputs{Module: "netcheck", Tree: "t1", Bases: []string{"src2:b"}, Contexts: map[string]string{"c": "x"},
Locks: map[string]string{"bin": "l1"}, Toolchains: map[string]delivery.Toolchain2{"bin": {Language: "go", Version: "1.26.0", Recipe: "7"}}}
}
// The form moved out of mesh-sdk computes what mesh-sdk's SourceFingerprint2 and NamedTree computed for the same
// inputs (golden values taken from mesh-sdk 3d87ddc), so the move itself rebuilds nothing.
func TestTheMovedComputationGivesTheValuesItGaveInMeshSDK(t *testing.T) {
in := base()
in.Packages = map[string]string{sdk: "abc"}
if fp, none := Of(in); fp != "src2:0f96be10d19712c94ea0ece8125a45cbb658ec06b705b28e747332cc94500e41" || none != "" {
t.Fatalf("with a package version: %q %q", fp, none)
}
if fp, _ := Of(base()); fp != "src2:f1a869faf14a3e422feba672156b42b1a7af7e7a1097c11554299f9e37e58625" {
t.Fatalf("without: %q", fp)
}
named, err := NamedTree(strings.Repeat("0", 64), map[string]string{"cmd": "t9", "go.mod": "b1"})
if err != nil || named != "named:0290bf64fc93b1c10954797d548be30d936008b041c58261faad0c388a3d4c80" {
t.Fatalf("named tree: %q %v", named, err)
}
in.Packages = map[string]string{sdk: ""}
if fp, none := Of(in); fp != "" || !strings.HasPrefix(none, NoPackageVersion) {
t.Fatalf("a package with no version: %q %q", fp, none)
}
}
func published(module, repo, dir, commit, requires string, digests map[string]string) delivery.PublishedPackages {
p := delivery.PublishedPackages{Module: module, Repository: repo, Dir: dir, Commit: commit, Requires: requires}
for _, pkg := range sortedKeys(digests) {
p.Packages = append(p.Packages, delivery.PackageDigest{Package: pkg, Digest: digests[pkg]})
}
return p
}
// ADR 0593 §2, §3: the fingerprint takes the digest of each package a module imports, and of each mesh Go module's
// requirements; a package it does not import moves nothing, an imported one or a sibling one moves it.
func TestTheImportFormMovesOnlyWithWhatTheModuleImports(t *testing.T) {
const sibling = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
imports := []string{sdk, sdk + "/announce", sibling}
pubs := func(announce, buildstep, sib, requires string) map[string]delivery.PublishedPackages {
return map[string]delivery.PublishedPackages{
sdk: published(sdk, "novox/mesh-sdk", "go", "c1", requires, map[string]string{sdk: "r", sdk + "/announce": announce, sdk + "/buildstep": buildstep}),
sibling: published(sibling, "novox/mesh-catalog", "modules/mesh-build-go/buildstep", "c9", "q", map[string]string{sibling: sib}),
}
}
fpOf := func(p map[string]delivery.PublishedPackages) string {
t.Helper()
r := Resolve("novox/mesh-catalog", imports, p)
if len(r.Missing) > 0 {
t.Fatalf("missing %v", r.Missing)
}
in := base()
in.Imports, in.Requires = r.Digests, r.Requires
fp, none := Of(in)
if fp == "" {
t.Fatal(none)
}
return fp
}
first := fpOf(pubs("a1", "b1", "s1", "q1"))
if got := fpOf(pubs("a1", "b2", "s1", "q1")); got != first {
t.Fatal("a package the module does not import (buildstep) moved its fingerprint")
}
if got := fpOf(pubs("a2", "b1", "s1", "q1")); got == first {
t.Fatal("an imported package (announce) changed and the fingerprint did not move")
}
if got := fpOf(pubs("a1", "b1", "s2", "q1")); got == first {
t.Fatal("a sibling package of the module's own repository changed and the fingerprint did not move")
}
if got := fpOf(pubs("a1", "b1", "s1", "q2")); got == first {
t.Fatal("mesh-sdk's requirements changed and the fingerprint did not move")
}
// Versions are pinned only for modules of another repository: a sibling is built from its directory.
r := Resolve("novox/mesh-catalog", imports, pubs("a1", "b1", "s1", "q1"))
if len(r.Versions) != 1 || r.Versions[sdk] != "c1" {
t.Fatalf("versions %v", r.Versions)
}
if r := Resolve("novox/mesh-catalog", append(imports, sdk+"/gone"), pubs("a1", "b1", "s1", "q1")); len(r.Missing) != 1 {
t.Fatalf("an import nobody publishes is missing: %v", r.Missing)
}
}
// A package's digest is its own files: a test file, a sub-package or a nested module changes nothing of it; a file
// of it, or of a sub-directory that is no package (testdata), changes only it.
func TestAPackageDigestIsItsOwnFiles(t *testing.T) {
files := map[string]string{
"go.mod": "m1", "go.sum": "s1", "stdio.go": "a", "stdio_test.go": "t",
"announce/announce.go": "b", "announce/testdata/x.json": "d",
"buildstep/step.go": "c", "buildstep/conformance/c.go": "e",
"tools/go.mod": "nm", "tools/t.go": "f", "internal/x/x.go": "g", "_old/o.go": "h",
}
requires, pkgs, err := Packages(sdk, files)
if err != nil {
t.Fatal(err)
}
digests := map[string]string{}
for _, p := range pkgs {
digests[p.Package] = p.Digest
}
want := []string{sdk, sdk + "/announce", sdk + "/buildstep", sdk + "/buildstep/conformance", sdk + "/internal/x"}
if got := sortedKeys(digests); strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("packages %v, want %v", got, want)
}
change := func(path, object string) map[string]string {
_, ps, err := Packages(sdk, with(files, path, object))
if err != nil {
t.Fatal(err)
}
out := map[string]string{}
for _, p := range ps {
out[p.Package] = p.Digest
}
return out
}
moved := func(after map[string]string) (out []string) {
for p, d := range after {
if digests[p] != d {
out = append(out, p)
}
}
return out
}
for path, want := range map[string]string{
"stdio_test.go": "",
"buildstep/conformance/c.go": sdk + "/buildstep/conformance",
"announce/testdata/x.json": sdk + "/announce",
"stdio.go": sdk,
"tools/t.go": "",
} {
got := moved(change(path, "zz"))
if (want == "" && len(got) != 0) || (want != "" && (len(got) != 1 || got[0] != want)) {
t.Errorf("changing %s moved %v, want %q", path, got, want)
}
}
if r2, _, _ := Packages(sdk, with(files, "go.sum", "s2")); r2 == requires {
t.Error("a changed go.sum left the requirements' digest")
}
if _, _, err := Packages(sdk, map[string]string{"a.go": "x"}); err == nil {
t.Error("a listing with no go.mod was taken as a module")
}
}
func with(m map[string]string, k, v string) map[string]string {
out := map[string]string{}
for a, b := range m {
out[a] = b
}
out[k] = v
return out
}
@@ -58,6 +58,16 @@ type Indexer struct {
JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r delivery.ManifestRefused)
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
// with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from.
// Nil publishes no package.
Files func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error)
// GoSum reads a module directory's go.sum at a commit (gitea.gitea_get_file): the mesh Go modules a module with no
// reported import set takes the version line of. Nil reads none: such a module keeps what it held.
GoSum func(ctx context.Context, repository, sha, dir string) ([]byte, error)
// Now is the clock of a taken value; nil is time.Now.
Now func() time.Time
Builds *Builds
@@ -80,19 +90,20 @@ type Indexer struct {
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
}
func (ix *Indexer) logf(format string, args ...any) {
@@ -0,0 +1,74 @@
package index
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
)
// form is a module's fingerprint, or why it has none, and the version of each mesh Go module of another repository
// its build is pinned to.
type form struct {
fp, none string
versions delivery.PackageVersions
}
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
// form they call for (novox/hq ADR 0593 §2, §4):
//
// - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is
// published (the packages state): the content digest of each imported package and the requirements' digest of each
// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only).
// - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs
// the build that reported the imports was built from. While the import form computes to it, nothing the build was
// built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions.
// Otherwise the import form is written, and ImportForm moves with it.
// - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module its
// go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), so its value is the one it had.
//
// It sets kept.ImportForm when it writes the import form.
func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, logf func(string, ...any)) form {
in := inputsFrom(module, *kept, toolchains)
if len(kept.Imports) > 0 {
r := fingerprint.Resolve(repository, kept.Imports, published)
if len(r.Missing) == 0 {
in.Imports, in.Requires = r.Digests, r.Requires
fp, none := fingerprint.Of(in)
switch {
case fp == "":
return form{none: none}
case kept.ImportForm == fp && prev.FP2 != "":
return form{fp: prev.FP2, versions: prev.Packages} // the build that reported the imports stands
}
kept.ImportForm = fp
return form{fp: fp, versions: r.Versions}
}
if logf != nil {
logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", "))
}
}
versions, unpublished := fingerprint.HeldVersionsOf(kept.Packages, held)
if len(unpublished) > 0 && logf != nil {
logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", "))
}
if len(kept.Packages) > 0 {
in.Packages = map[string]string{}
for _, p := range kept.Packages {
in.Packages[p] = versions[p]
}
}
fp, none := fingerprint.Of(in)
if fp == "" {
return form{none: none}
}
return form{fp: fp, versions: versions}
}
// inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of
// either form yet.
func inputsFrom(module string, k delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2) fingerprint.Inputs {
return fingerprint.Inputs{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts,
Locks: k.Locks, Toolchains: toolchains}
}
@@ -0,0 +1,134 @@
package index
import (
"context"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
)
// ApplyBuildEnd keeps the imports a done build reports at its end (delivery.BuildLogEntry.Imports; novox/hq ADR 0593
// §2) on the module it built, when the build's fingerprint is the one the module holds: the import set of the module's
// last trunk build. With them it keeps ImportForm, the import-form fingerprint of exactly what the build was built from:
// the module's kept inputs and the toolchains its fingerprint was computed with, and each imported package's digest at
// the version the build took it at (another repository's at the version the module's fingerprint pins, its own
// repository's at the commit built). A recompute that gives that value again keeps the fingerprint (§4); one that
// differs writes the import form. The fingerprint itself is not changed here.
//
// An entry that is no done build's end, reports no imports, or is of a fingerprint the module no longer holds is left.
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
// resolves them writes the import form, one rebuild too many, never a stale module.
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
e, err := delivery.DecodeBuildLogEntry(value)
if err != nil {
return err
}
if e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
return nil
}
key, err := delivery.ModulesKey(e.Module)
if err != nil {
return nil
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
raw, _, found, err := ix.Modules.Get(ctx, key)
if err != nil || !found {
return err
}
cur, err := delivery.DecodeModule(raw)
if err != nil || cur.FP2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
}
if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" {
return nil
}
_, m, err := ix.manifestRaw(ctx, cur.Manifest)
if err != nil {
return fmt.Errorf("modules.%s: its manifest: %w", key, err)
}
toolchains := map[string]delivery.Toolchain2{}
for artifact, language := range languagesOf(m) {
used, ok := cur.Toolchains[language]
if !ok {
ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language)
return nil
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
kept := *cur.Inputs2
kept.Imports, kept.ImportForm = slices.Clone(e.Imports), ""
if built, err := ix.publishedAsBuilt(ctx, cur, e); err != nil {
ix.logf("%s: the packages its build imported cannot be read as built (%v): its imports are kept, and its next recompute writes the import form", e.Module, err)
} else if r := fingerprint.Resolve(cur.Repository, e.Imports, built); len(r.Missing) > 0 {
ix.logf("%s: no published package holds %v: its imports are kept, and the recompute that resolves them writes the import form", e.Module, r.Missing)
} else {
in := inputsFrom(e.Module, kept, toolchains)
in.Imports, in.Requires = r.Digests, r.Requires
if fp, _ := fingerprint.Of(in); fp != "" {
kept.ImportForm = fp
}
}
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
now, err := delivery.DecodeModule(value)
if err != nil || now.FP2 != e.FP || now.Manifest != cur.Manifest || now.Inputs2 == nil {
return nil, false, nil
}
in := *now.Inputs2
in.Imports, in.ImportForm = kept.Imports, kept.ImportForm
now.Inputs2 = &in
data, err := delivery.EncodeModule(now)
return data, false, err
})
}
// publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took
// it: a module of the built module's own repository at the commit built, one of another repository at the version the
// module's fingerprint pins (cur.Packages), else its latest published version. A module whose published commit is the
// one built is taken from the packages state; any other is read from the forge's listing at that commit.
func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e delivery.BuildLogEntry) (map[string]delivery.PublishedPackages, error) {
published := ix.publishedAll(ctx)
if r := fingerprint.Resolve(cur.Repository, e.Imports, published); len(r.Missing) > 0 {
if err := ix.FillPackages(ctx); err != nil {
return nil, err
}
published = ix.publishedAll(ctx)
}
type at struct{ repository, commit string }
reread := map[at][]string{}
for _, imp := range e.Imports {
r := fingerprint.Resolve(cur.Repository, []string{imp}, published)
for mod := range r.Requires {
p := published[mod]
commit := p.Commit
switch {
case delivery.SameRepository(p.Repository, cur.Repository) && e.Commit != "":
commit = e.Commit
case cur.Packages[mod] != "":
commit = cur.Packages[mod]
}
if commit != p.Commit {
k := at{p.Repository, commit}
if !slices.Contains(reread[k], mod) {
reread[k] = append(reread[k], mod)
}
}
}
}
out := maps.Clone(published)
for k, mods := range reread {
got, err := ix.packagesAt(ctx, k.repository, k.commit, mods, nil)
if err != nil {
return nil, err
}
maps.Copy(out, got)
}
return out, nil
}
@@ -0,0 +1,155 @@
package index
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const meshSDK = "git.novox.be/novox/mesh-sdk/go"
// Full commits of mesh-sdk and of the catalogue.
var (
c1 = strings.Repeat("1", 40)
c2 = strings.Repeat("2", 40)
c3 = strings.Repeat("3", 40)
cat1 = strings.Repeat("a", 40)
)
// sdkFiles is a listing of mesh-sdk's go directory (gitea_commit_trees with files): announce and buildstep at the
// given blobs.
func sdkFiles(announce, buildstep string) map[string]string {
return map[string]string{"go.mod": "m1", "go.sum": "s1", "stdio.go": "r1", "announce/announce.go": announce, "buildstep/step.go": buildstep}
}
// forge answers Files from the listing of each commit of mesh-sdk.
type forge map[string]map[string]string
func (f forge) files(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
out := map[string]map[string]string{}
for _, d := range dirs {
if repository != "novox/mesh-sdk" || d != "go" || f[sha] == nil {
return nil, &missingDir{d}
}
out[d] = f[sha]
}
return out, nil
}
type missingDir struct{ dir string }
func (m *missingDir) Error() string { return "no directory " + m.dir }
// ADR 0593 §2-§4 (D1, D2, D3): a build's imports are kept with no fingerprint moving; a mesh-sdk merge that changes
// a package the module does not import leaves it; one that changes an imported package moves it to the import form;
// and the packages state holds one key with a digest per package.
func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T) {
ctx := context.Background()
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
listings := forge{c1: sdkFiles("a1", "b1"), c2: sdkFiles("a1", "b2"), c3: sdkFiles("a2", "b2")}
ix.Packages, ix.Files = newMem(), listings.files
// avahi's go.sum names mesh-sdk (the version line), as every Go module's does; mesh-sdk's module is held at c1.
setPackages(t, mods, "avahi", []string{meshSDK})
sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
if err != nil {
t.Fatal(err)
}
mods.put("mesh-sdk", sdkModule)
held, _ := ix.heldModules(ctx)
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil {
t.Fatal(err)
}
pub := ix.publishedAll(ctx)[meshSDK]
if pub.Commit != c1 || len(pub.Packages) != 3 || pub.Requires == "" {
t.Fatalf("packages.%s = %+v, want commit c1 and a digest per package (go, announce, buildstep)", meshSDK, pub)
}
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
before, _ := moduleOf(t, mods, "avahi")
if before.FP2 == "" {
t.Fatalf("no fingerprint to start from: %q", before.None2)
}
// The build of that fingerprint ends, reporting its imports: kept, and the fingerprint does not move.
end := buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce")
if err := ix.ApplyBuildEnd(ctx, end); err != nil {
t.Fatal(err)
}
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
after, _ := moduleOf(t, mods, "avahi")
if after.FP2 != before.FP2 || strings.Join(after.Inputs2.Imports, ",") != meshSDK+","+meshSDK+"/announce" || after.Inputs2.ImportForm == "" {
t.Fatalf("after the build's imports: fp2 %q (was %q), imports %v, import form %q; want the same fingerprint and the imports kept",
after.FP2, before.FP2, after.Inputs2.Imports, after.Inputs2.ImportForm)
}
// mesh-sdk merges a change of buildstep only: avahi does not import it.
held, _ = ix.heldModules(ctx)
if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false); err != nil {
t.Fatal(err)
}
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "avahi"); m.FP2 != before.FP2 {
t.Fatalf("a mesh-sdk merge changing only buildstep moved avahi's fingerprint to %q", m.FP2)
}
// mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version.
held, _ = ix.heldModules(ctx)
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil {
t.Fatal(err)
}
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
moved, _ := moduleOf(t, mods, "avahi")
if moved.FP2 == before.FP2 || moved.FP2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3",
moved.FP2, before.FP2, moved.Inputs2.ImportForm, moved.Packages)
}
}
// A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept.
func TestABuildOfAnOlderFingerprintKeepsNoImports(t *testing.T) {
ctx := context.Background()
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
if err := ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK)); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || len(m.Inputs2.Imports) != 0 {
t.Fatalf("imports %v kept from a build of another fingerprint", m.Inputs2)
}
}
func buildEnd(t *testing.T, module, fp string, imports ...string) []byte {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: fp,
Repository: catalogueRepo, Directory: "modules/" + module, Commit: cat1, Ref: "main", Node: "n1", At: time.Now(),
Outcome: delivery.BuildDone, Imports: imports})
if err != nil {
t.Fatal(err)
}
return raw
}
// setPackages writes a module's version-line packages, as a merge reading its go.sum keeps them.
func setPackages(t *testing.T, mods *memStore, module string, packages []string) {
t.Helper()
m, _ := moduleOf(t, mods, module)
in := *m.Inputs2
in.Packages = packages
m.Inputs2 = &in
raw, err := delivery.EncodeModule(m)
if err != nil {
t.Fatal(err)
}
mods.put(module, raw)
}
@@ -10,6 +10,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/judge"
)
@@ -55,7 +56,7 @@ type baseFP func(module string) (fp, none string)
// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
// resolves packages when it is built (wantsPackages, :1017) only when its language's lock file exists: it then has
// an entry in Locks with that file's hash, and an artifact whose lock file the merge did not carry has no entry (a
// dependency-free or vendored bundle resolves nothing, as mesh-sdk SourceInputs2 says of an artifact not in Locks;
// dependency-free or vendored bundle resolves nothing, as fingerprint.Inputs says of an artifact not in Locks;
// novox/hq ADR 0469 clause 5, issue 1459). A bundle that needs a lock and lacks one is refused where it is built, not
// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve
// nothing and have no toolchain. A base is a build.on entry that names a module.
@@ -65,7 +66,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo
// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery
// Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its
// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs
// manifest, by the indexer's one function (fingerprint.NamedTree), and keeps their objects, so a computation from kept inputs
// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it
// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given.
//
@@ -104,7 +105,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, p
if len(missing) > 0 {
return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths}
}
if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
if tree, err = fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths}
}
}
@@ -9,6 +9,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
"mesh-delivery-indexer/judge"
)
@@ -97,9 +98,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool,
if next.Manifest == "" {
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
} else {
packages := delivery.OtherPackages(f.Packages[dir], f.Repository) // its own repository's come with its tree
packages := ix.goSumPackages(ctx, f, dir, cur, has)
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept != nil && has && cur.Inputs2 != nil {
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
}
if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
// that reads it judges the module again at start with no merge (issue 1923).
@@ -118,6 +123,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool,
return w, err
}
}
// The packages this merge publishes come first, so the modules importing a changed one move in this recompute (ADR
// 0593 §3). A forge that cannot list them leaves the state as it was, said: the importers move at the next merge.
reach := maps.Clone(held)
maps.Copy(reach, overlay)
if err := ix.publish(ctx, f.Repository, f.Merge, reach, false); err != nil {
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
}
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return w, ix.recomputeWith(ctx, overlay, w)
}
@@ -320,3 +332,39 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
in.Packages = slices.Clone(packages)
return in
}
// goSumPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names (the version
// line's packages, ADR 0573 §2), read by the indexer itself (the bridge reads none since ADR 0593 §2): those the message
// carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read from the
// forge (GoSum). A directory with no go.sum has none.
func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
if carried := f.Packages[dir]; len(carried) > 0 {
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
}
hash := f.Locks[dir]["go.sum"]
if hash == "" {
return nil
}
if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) {
for _, h := range cur.Inputs2.Locks {
if h == hash {
return cur.Inputs2.Packages
}
}
}
if ix.GoSum == nil {
if has && cur.Inputs2 != nil {
return cur.Inputs2.Packages
}
return nil
}
raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir)
if err != nil {
ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
if has && cur.Inputs2 != nil {
return cur.Inputs2.Packages
}
return nil
}
return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository)
}
@@ -0,0 +1,326 @@
package index
import (
"context"
"fmt"
"maps"
"path"
"reflect"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
)
// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
// digests in its state `packages` (delivery.PackagesBucket), one key per Go module. The builder, the checker and the
// build modules read it there; nobody else computes a digest or a version.
//
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
// reported (KeptInputs2.Imports). Its key is written at each merge of its repository, before the merge's recompute, so
// a merge that changes a package moves its importers in the same recompute (§3); and once at the start, for each
// reached module the state does not hold yet (FillPackages).
// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.PublishedPackages {
out := map[string]delivery.PublishedPackages{}
if ix.Packages == nil {
return out
}
keys, err := ix.Packages.Keys(ctx)
if err != nil {
ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err)
return out
}
for _, key := range keys {
value, _, found, err := ix.Packages.Get(ctx, key)
if err != nil || !found {
continue
}
p, err := delivery.DecodePublishedPackages(value)
if err != nil {
ix.logf("packages.%s holds what is not a published module, left out: %v", key, err)
continue
}
out[p.Module] = p
}
return out
}
// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
// reported.
func reached(held map[string]delivery.Module) (modules, imports map[string][]string) {
modules, imports = map[string][]string{}, map[string][]string{}
add := func(to map[string][]string, p string) {
repo, _, ok := delivery.PackageRepository(p)
if ok && !slices.Contains(to[repo], p) {
to[repo] = append(to[repo], p)
}
}
for _, m := range held {
if m.Inputs2 == nil {
continue
}
for _, p := range m.Inputs2.Packages {
add(modules, p)
}
for _, p := range m.Inputs2.Imports {
add(imports, p)
}
}
for _, by := range []map[string][]string{modules, imports} {
for r := range by {
slices.Sort(by[r])
}
}
return modules, imports
}
// packagesAt are the Go modules of repository at commit that hold modules (module paths) or imports (import paths),
// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
// directory by directory). A module the listing has no go.mod for is said and left out.
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]delivery.PublishedPackages, error) {
if ix.Files == nil || commit == "" {
return nil, nil
}
dirOf := map[string]string{} // module path to its directory in the repository
for _, m := range modules {
if _, dir, ok := delivery.PackageRepository(m); ok {
dirOf[m] = dir
}
}
holds := func(imp string) bool {
for m := range dirOf {
if imp == m || strings.HasPrefix(imp, m+"/") {
return true
}
}
return false
}
listings := map[string]map[string]string{} // by directory
list := func(dirs []string) error {
var ask []string
for _, d := range dirs {
if _, done := listings[d]; !done {
ask = append(ask, d)
}
}
if len(ask) == 0 {
return nil
}
got, err := ix.Files(ctx, repository, commit, ask)
if err != nil {
return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err)
}
for _, d := range ask {
listings[d] = got[d]
}
return nil
}
for _, imp := range imports {
if holds(imp) {
continue
}
_, dir, _ := delivery.PackageRepository(imp)
for d := dir; ; d = parent(d) {
if err := list([]string{d}); err != nil {
return nil, err
}
if _, ok := listings[d]["go.mod"]; ok {
m := strings.TrimSuffix(delivery.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
dirOf[m] = d
break
}
if d == "" {
ix.logf("%s at %s: no go.mod holds the import %s, so it has no digest", repository, commit, imp)
break
}
}
}
if err := list(slices.Sorted(maps.Values(dirOf))); err != nil {
return nil, err
}
out := map[string]delivery.PublishedPackages{}
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
dir := dirOf[m]
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
if err != nil {
ix.logf("%s at %s: %v; it is not published", repository, commit, err)
continue
}
d := dir
if d == "" {
d = delivery.RootPath
}
out[m] = delivery.PublishedPackages{Module: m, Repository: delivery.RepositoryName(repository), Dir: d, Commit: commit,
Requires: requires, Packages: pkgs}
}
return out, nil
}
// parent is the directory above d, "" for the repository's root.
func parent(d string) string {
if d == "" {
return ""
}
p := path.Dir(d)
if p == "." {
return ""
}
return p
}
// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]delivery.Module, onlyMissing bool) error {
if ix.Packages == nil || ix.Files == nil {
return nil
}
modules, imports := reached(held)
repo := delivery.RepositoryName(repository)
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
return nil
}
have := ix.publishedAll(ctx)
if onlyMissing {
// Imports a held module already resolves are not asked again.
var ms, is []string
for _, m := range modules[repo] {
if _, ok := have[m]; !ok {
ms = append(ms, m)
}
}
for _, imp := range imports[repo] {
if len(fingerprint.Resolve(repo, []string{imp}, have).Missing) > 0 {
is = append(is, imp)
}
}
if len(ms) == 0 && len(is) == 0 {
return nil
}
modules[repo], imports[repo] = ms, is
}
// An import a published module of the repository already holds is resolved by its module's directory.
for m, p := range have {
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
modules[repo] = append(modules[repo], m)
}
}
got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo])
if err != nil {
return err
}
for _, m := range slices.Sorted(maps.Keys(got)) {
next := got[m]
key, err := delivery.PackagesKey(m)
if err != nil {
ix.logf("%s: %v", m, err)
continue
}
err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) {
return nil, false, nil
}
}
next.At = ix.now().UTC()
data, err := delivery.EncodePublishedPackages(next)
return data, false, err
})
if err != nil {
return fmt.Errorf("packages.%s: %w", key, err)
}
}
return nil
}
// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
// A repository with no taken merge is left for its next merge.
func (ix *Indexer) FillPackages(ctx context.Context) error {
if ix.Packages == nil || ix.Files == nil || ix.Modules == nil || ix.Taken == nil {
return nil
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
modules, imports := reached(held)
repos := map[string]bool{}
for r := range modules {
repos[r] = true
}
for r := range imports {
repos[r] = true
}
for _, repo := range slices.Sorted(maps.Keys(repos)) {
key, err := delivery.RepositoryKey(repo)
if err != nil {
continue
}
value, _, found, err := ix.Taken.Get(ctx, key)
if err != nil || !found {
continue
}
t, err := delivery.DecodeTaken(value)
if err != nil {
continue
}
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
}
}
return nil
}
// candidatePublished is the packages state as a candidate builds against it (novox/hq ADR 0573 §4): each Go module a
// member's repository publishes at the member's head (members, by repository), every other at its latest published
// version. A head's modules are read from the forge once and kept while the head is a member somewhere; one the forge
// cannot list is said and left at its published version.
func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]delivery.Module) map[string]delivery.PublishedPackages {
out := ix.publishedAll(ctx)
if len(members) == 0 {
return out
}
modules, imports := reached(held)
for _, repo := range slices.Sorted(maps.Keys(members)) {
head := members[repo]
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
continue
}
at := repo + "@" + head
ix.mu.Lock()
got, ok := ix.atHeads[at]
ix.mu.Unlock()
if !ok {
var ms []string
ms = append(ms, modules[repo]...)
for m, p := range out {
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) {
ms = append(ms, m)
}
}
var err error
if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil {
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
continue
}
ix.mu.Lock()
if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
}
ix.atHeads[at] = got
ix.mu.Unlock()
}
maps.Copy(out, got)
}
return out
}
// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again.
const maxHeadsKept = 64
@@ -9,6 +9,8 @@ import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
)
// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull
@@ -254,7 +256,15 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = NonePreviewNoManifest
return out
}
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, delivery.OtherPackages(pm.Packages, pv.Repository), ix.baseFP(ctx))
// The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer
// computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build
// module still names them.
trunk := held[m]
packages := fingerprint.OtherPackages(pm.Packages, pv.Repository)
if len(pm.Packages) == 0 && trunk.Inputs2 != nil {
packages = trunk.Inputs2.Packages
}
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
if in.Kept == nil {
out.None2 = in.None2
return out
@@ -274,8 +284,29 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
toolchains[artifact] = t.tc
}
inputs := *in.Kept // no builder input (ADR 0530 §1)
versions := candidatePackages(inputs.Packages, held, members)
fp, none := delivery.SourceFingerprint2(inputs.WithPackages(m, toolchains, versions))
var fp, none string
var versions delivery.PackageVersions
if trunk.Inputs2 != nil && len(trunk.Inputs2.Imports) > 0 {
inputs.Imports, inputs.ImportForm = trunk.Inputs2.Imports, trunk.Inputs2.ImportForm
}
// The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk
// computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed.
published := ix.candidatePublished(ctx, members, held)
if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 {
f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, nil)
fp, none, versions = f.fp, f.none, f.versions
} else {
inputs.Imports, inputs.ImportForm = nil, ""
versions = candidatePackages(inputs.Packages, held, members)
in := inputsFrom(m, inputs, toolchains)
if len(inputs.Packages) > 0 {
in.Packages = map[string]string{}
for _, p := range inputs.Packages {
in.Packages[p] = versions[p]
}
}
fp, none = fingerprint.Of(in)
}
if fp == "" {
out.None2 = none
return out
@@ -291,7 +322,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
// version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none.
func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions {
versions, _ := delivery.PackageVersionsOf(packages, held)
versions, _ := fingerprint.HeldVersionsOf(packages, held)
for _, p := range packages {
repository, _, _ := delivery.PackageRepository(p)
if head := members[repository]; head != "" {
@@ -199,6 +199,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
}
tools := ix.agreed()
published := ix.publishedAll(ctx)
type result struct {
fp, none string
tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
@@ -254,12 +255,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
}
toolchains[artifact] = t.tc
}
// The latest published version of each mesh package the module uses: the trunk merge of the module that holds it,
// as this recompute holds it, so a package's merge moves every dependent in the same recompute (ADR 0573 §2).
versions, unpublished := delivery.PackageVersionsOf(kept.Packages, held)
if len(unpublished) > 0 {
ix.logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", "))
}
var r result
switch {
case k.contexts:
@@ -268,13 +263,14 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
if fp, ok := stillComputed(module, kept, k, toolchains, versions); ok {
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: versions}
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, published); ok {
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
}
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, published, ix.logf)
r.fp, r.none = f.fp, f.none
if r.fp != "" {
r.tcs, r.pkgs = toolchainsUsed(toolchains), versions
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
}
}
done[module] = r
@@ -421,6 +417,7 @@ func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur deliv
if in.Kept == nil {
return cur, false
}
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
cur.Inputs2, cur.None2, cur.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
return cur, true
}
@@ -442,11 +439,12 @@ func hasContext(m catalogue.Manifest) bool {
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, versions delivery.PackageVersions) (string, bool) {
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
// module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages) (string, delivery.PackageVersions, bool) {
if k.held.FP2 == "" {
return "", false
return "", nil, false
}
toolchains := map[string]delivery.Toolchain2{}
for artifact, language := range k.languages {
@@ -456,12 +454,12 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma
}
used, ok := k.held.Toolchains[language]
if !ok {
return "", false
return "", nil, false
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
fp, none := delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
return fp, none == "" && fp == k.held.FP2
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, nil)
return f.fp, f.versions, f.none == "" && f.fp == k.held.FP2
}
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
@@ -11,6 +11,8 @@ import (
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
@@ -87,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("emits = %v, want manifest-refused alone", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
+2 -1
View File
@@ -20,7 +20,8 @@
{ "name": "config-candidates", "history": 5 },
{ "name": "taken", "history": 1 },
{ "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 }
{ "name": "verdicts", "history": 5 },
{ "name": "packages", "history": 5 }
],
"emits": [
"manifest-refused"