Indexer computes fingerprints alone, over the packages a module imports
The fingerprint, named tree and package versions move from mesh-sdk into the indexer's internal/fingerprint (ADR 0593 §2). The indexer publishes each reached mesh Go module's package digests from the forge's listing in its new packages state, keeps the imports a done build reports, and writes the import form only when an input differs from the build that reported them (§4), so the form change costs no rebuild and a package merge moves only its importers (§3). Candidates take the same form against the group's member heads (D1-D4). Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
@@ -293,14 +293,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer calls.Close()
|
||||
// The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears.
|
||||
verdicts := process.AwaitBucket(ctx, js, conn, index.VerdictsBucket, "no verdict on a manifest is written", logger.Printf)
|
||||
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
|
||||
// fingerprint takes the version line.
|
||||
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call)}
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call)}
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
|
||||
if err := calls.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
|
||||
if err := known.Put(le.Value); err != nil {
|
||||
logger.Printf("%s entry %d is not read: %v", delivery.BuildLogRead, le.Sequence, err)
|
||||
@@ -310,6 +314,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil {
|
||||
logger.Printf("the registered module %s is not indexed: %v", le.Key, err)
|
||||
}
|
||||
// A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are.
|
||||
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
|
||||
}
|
||||
} else {
|
||||
startEnds = append(startEnds, le)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
@@ -364,6 +374,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as
|
||||
// having no toolchain reported, and back).
|
||||
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
|
||||
// the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4).
|
||||
packages.Ready(func(jetstream.KeyValue) {
|
||||
if err := ix.FillPackages(ctx); err != nil {
|
||||
logger.Printf("publishing the packages at start: %v", err)
|
||||
}
|
||||
})
|
||||
for _, le := range startEnds {
|
||||
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", delivery.BuildLogRead, le.Sequence, err)
|
||||
}
|
||||
}
|
||||
startEnds = nil
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
@@ -78,6 +79,65 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string)
|
||||
}
|
||||
}
|
||||
|
||||
// filesVia asks the forge for every blob under each directory at a commit (the same tool with `files`, ADR 0593 §2): it
|
||||
// answers {files} by directory, then path relative to it, each with its blob's object id. An answer that lists no
|
||||
// directory asked is an error naming it; a cut listing fails the call in the tool.
|
||||
func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "files": dirs})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Files map[string]map[string]string `json:"files"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {files}: %w", commitTreesTool, err)
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if _, ok := a.Files[d]; !ok {
|
||||
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
|
||||
}
|
||||
}
|
||||
return a.Files, nil
|
||||
}
|
||||
}
|
||||
|
||||
// goSumVia reads a module directory's go.sum at a commit (gitea_get_file): the version line's packages of a module
|
||||
// whose build reported no imports (ADR 0593 §2). A directory with no go.sum answers nothing.
|
||||
func goSumVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
|
||||
return func(_ context.Context, repository, sha, dir string) ([]byte, error) {
|
||||
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
p := "go.sum"
|
||||
if dir != "" && dir != delivery.RootPath {
|
||||
p = dir + "/go.sum"
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), ": 404 ") {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var f struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return []byte(*f.File.Content), nil
|
||||
}
|
||||
}
|
||||
|
||||
// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor).
|
||||
func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) {
|
||||
return func(_ context.Context, repository, commit, in string) (bool, error) {
|
||||
@@ -166,7 +226,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
}
|
||||
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||
if name == "go.sum" {
|
||||
if p := delivery.MeshPackagesOf([]byte(c)); len(p) > 0 {
|
||||
if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 {
|
||||
if out.Packages == nil {
|
||||
out.Packages = map[string][]string{}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
// Package fingerprint is the one computation of a module's source fingerprint, of a named tree, of a mesh package's
|
||||
// content digest and of a package's published version (novox/hq ADR 0593 §2): the indexer's own, and nobody else's. It
|
||||
// left mesh-sdk's delivery package, which keeps only the fingerprint's form (delivery.SourcePrefix2, the key part) and
|
||||
// the shapes the indexer writes (delivery.Module, delivery.PublishedPackages). No other part computes any of it: the
|
||||
// builder, the checker and the build modules read what the indexer writes.
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// NoPackageVersion opens the reason a module has no fingerprint while a mesh package it uses has no published version
|
||||
// the indexer holds (novox/hq ADR 0573 §2); the packages follow, by module path.
|
||||
const NoPackageVersion = "no published version of the package "
|
||||
|
||||
// Inputs are what a module's source fingerprint is of (novox/hq ADR 0429 clause 5 as ADR 0437 clause 5 changed it; ADR
|
||||
// 0453 clause 6; ADR 0593 §2). A base enters by its module's fingerprint, an artifact that resolves packages when it is
|
||||
// built by the hash of its lock file (an entry with an empty hash is such an artifact with no lock, which has no
|
||||
// fingerprint), an artifact's toolchain as delivery.Toolchain2.
|
||||
//
|
||||
// The mesh packages enter in one of two forms. **The import form** (ADR 0593 §2): Imports, the content digest of each
|
||||
// mesh package the module's programs import (by import path), and Requires, the digest of each mesh Go module's go.mod
|
||||
// and go.sum (by module path). **The version line** (ADR 0573 §2), for a module with no reported import set:
|
||||
// Packages, the published version of each mesh Go module its lock file names. A caller gives one form, never both.
|
||||
type Inputs struct {
|
||||
Module string
|
||||
Tree string
|
||||
Bases []string
|
||||
Builder string
|
||||
Contexts map[string]string
|
||||
Locks map[string]string
|
||||
Toolchains map[string]delivery.Toolchain2
|
||||
Packages map[string]string
|
||||
Imports map[string]string
|
||||
Requires map[string]string
|
||||
Unpinned string
|
||||
}
|
||||
|
||||
// Of is the module's source fingerprint, `src2:` and the hex sha256 of its inputs' lines, or none and the reason when
|
||||
// the source does not pin the build: Unpinned when set, "no tree" when Tree is empty, "no lock" when an artifact
|
||||
// resolves packages and has no lock, NoPackageVersion when a package of the version line has none. It is a pure
|
||||
// function of its inputs. The lines are "module <m>", "tree <t>" and "builder <fp>", then, sorted, "base <fp>" (once
|
||||
// each), "context <artifact> <tree>", "lock <artifact> <hash>", "toolchain <artifact> <language> <version>
|
||||
// <recipe>", "package <module> <version>", "import <package> <digest>" and "requires <module> <digest>", joined by a
|
||||
// newline. Inputs with no package of either form give the value they gave before packages entered.
|
||||
func Of(in Inputs) (fp, none string) {
|
||||
if in.Unpinned != "" {
|
||||
return "", in.Unpinned
|
||||
}
|
||||
if in.Tree == "" {
|
||||
return "", "no tree"
|
||||
}
|
||||
for _, h := range in.Locks {
|
||||
if h == "" {
|
||||
return "", "no lock"
|
||||
}
|
||||
}
|
||||
var unpublished []string
|
||||
for p, v := range in.Packages {
|
||||
if v == "" {
|
||||
unpublished = append(unpublished, p)
|
||||
}
|
||||
}
|
||||
if len(unpublished) > 0 {
|
||||
sort.Strings(unpublished)
|
||||
return "", NoPackageVersion + strings.Join(unpublished, ", ")
|
||||
}
|
||||
lines := []string{"module " + in.Module, "tree " + in.Tree, "builder " + in.Builder}
|
||||
bases := map[string]bool{}
|
||||
for _, b := range in.Bases {
|
||||
bases[b] = true
|
||||
}
|
||||
for b := range bases {
|
||||
lines = append(lines, "base "+b)
|
||||
}
|
||||
for a, t := range in.Contexts {
|
||||
lines = append(lines, "context "+a+" "+t)
|
||||
}
|
||||
for a, h := range in.Locks {
|
||||
lines = append(lines, "lock "+a+" "+h)
|
||||
}
|
||||
for a, t := range in.Toolchains {
|
||||
lines = append(lines, "toolchain "+a+" "+t.Language+" "+t.Version+" "+t.Recipe)
|
||||
}
|
||||
for p, v := range in.Packages {
|
||||
lines = append(lines, "package "+p+" "+v)
|
||||
}
|
||||
for p, d := range in.Imports {
|
||||
lines = append(lines, "import "+p+" "+d)
|
||||
}
|
||||
for m, d := range in.Requires {
|
||||
lines = append(lines, "requires "+m+" "+d)
|
||||
}
|
||||
sort.Strings(lines[3:])
|
||||
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
return delivery.SourcePrefix2 + hex.EncodeToString(sum[:]), ""
|
||||
}
|
||||
|
||||
// NamedTree is the tree input of a module whose every artifact names the paths it is built from (novox/hq ADR 0499
|
||||
// clause 2): `named:` and the hex sha256 of the lines "manifest <digest>" and, sorted, "path <path> <object>", joined by
|
||||
// a newline. manifest is the module's delivery.ManifestDigest; paths are the named paths, each clean and relative to
|
||||
// the module's directory, with the git object id each has at the commit. It refuses no path, a named path with no
|
||||
// object, a path that is not clean, and a manifest that is not a digest, naming them: the module then has none.
|
||||
func NamedTree(manifest string, paths map[string]string) (string, error) {
|
||||
if _, err := delivery.ManifestsKey(manifest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return "", fmt.Errorf("the module names no path, so it has no named tree")
|
||||
}
|
||||
lines := make([]string, 0, len(paths))
|
||||
for p, object := range paths {
|
||||
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
|
||||
return "", fmt.Errorf("the named path %q is not a clean path inside the module's directory", p)
|
||||
}
|
||||
if object == "" {
|
||||
return "", fmt.Errorf("the named path %q has no git object at the commit", p)
|
||||
}
|
||||
lines = append(lines, "path "+p+" "+object)
|
||||
}
|
||||
sort.Strings(lines)
|
||||
sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n")))
|
||||
return delivery.NamedTreePrefix + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under
|
||||
// delivery.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
|
||||
// takes the version line of (ADR 0593 §2).
|
||||
func MeshPackagesOf(goSum []byte) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(string(goSum), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := delivery.PackageRepository(f[0]); ok {
|
||||
seen[f[0]] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// OtherPackages are the packages not published from repository, the module's own, in their order; nil for none.
|
||||
func OtherPackages(packages []string, repository string) []string {
|
||||
var out []string
|
||||
for _, p := range packages {
|
||||
if repo, _, ok := delivery.PackageRepository(p); ok && !delivery.SameRepository(repo, repository) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Resolved is an import set resolved against the published packages: the digest of each import, the requirements'
|
||||
// digest of each mesh Go module holding one, the version to build each of another repository's against, and the
|
||||
// imports none publishes.
|
||||
type Resolved struct {
|
||||
Digests map[string]string
|
||||
Requires map[string]string
|
||||
Versions delivery.PackageVersions
|
||||
Missing []string
|
||||
}
|
||||
|
||||
// Resolve resolves the imports of a module of repository against the published packages (by module path): each
|
||||
// import is held by the published module with the longest path that publishes it. A module of repository itself is
|
||||
// built from its directory, so it is given no version (a same-repository replace, ADR 0593 §1); it still enters by
|
||||
// its digests.
|
||||
func Resolve(repository string, imports []string, published map[string]delivery.PublishedPackages) Resolved {
|
||||
r := Resolved{Digests: map[string]string{}, Requires: map[string]string{}}
|
||||
for _, imp := range imports {
|
||||
var holder *delivery.PublishedPackages
|
||||
digest := ""
|
||||
for _, m := range slices.Sorted(maps.Keys(published)) {
|
||||
p := published[m]
|
||||
if imp != m && !strings.HasPrefix(imp, m+"/") {
|
||||
continue
|
||||
}
|
||||
d, ok := p.DigestOf(imp)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if holder == nil || len(m) > len(holder.Module) {
|
||||
holder, digest = &p, d
|
||||
}
|
||||
}
|
||||
if holder == nil {
|
||||
r.Missing = append(r.Missing, imp)
|
||||
continue
|
||||
}
|
||||
r.Digests[imp] = digest
|
||||
r.Requires[holder.Module] = holder.Requires
|
||||
if !delivery.SameRepository(holder.Repository, repository) {
|
||||
if r.Versions == nil {
|
||||
r.Versions = delivery.PackageVersions{}
|
||||
}
|
||||
r.Versions[holder.Module] = holder.Commit
|
||||
}
|
||||
}
|
||||
slices.Sort(r.Missing)
|
||||
return r
|
||||
}
|
||||
|
||||
// Packages are a mesh Go module's requirements' digest and the content digest of each of its packages, from files,
|
||||
// every blob under the module's directory by path relative to it with its git object id (the forge's listing,
|
||||
// gitea_commit_trees with files). It refuses a listing with no go.mod at the module's root.
|
||||
//
|
||||
// A package is a directory that directly holds a .go file that is not a test (`testdata`, and a directory whose name
|
||||
// starts with `_` or `.`, is none, and nor is anything under a directory holding a go.mod of its own, another
|
||||
// module). Its digest is the hex sha256 of the sorted lines "<path relative to it> <object>" of every file under it,
|
||||
// leaving out `*_test.go` and everything under a sub-directory that is itself a package or holds a go.mod: a file
|
||||
// //go:embed can reach counts, a test or a sub-package does not. The requirements' digest is the hex sha256 of the
|
||||
// lines "go.mod <object>" and "go.sum <object>" (the second absent with no go.sum).
|
||||
func Packages(module string, files map[string]string) (requires string, packages []delivery.PackageDigest, err error) {
|
||||
goMod, ok := files["go.mod"]
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("the module %s has no go.mod at its directory", module)
|
||||
}
|
||||
req := "go.mod " + goMod
|
||||
if goSum, ok := files["go.sum"]; ok {
|
||||
req += "\ngo.sum " + goSum
|
||||
}
|
||||
sum := sha256.Sum256([]byte(req))
|
||||
requires = hex.EncodeToString(sum[:])
|
||||
|
||||
nested := map[string]bool{} // directories holding a go.mod of their own
|
||||
for p := range files {
|
||||
if dir, name := path.Split(p); name == "go.mod" && dir != "" {
|
||||
nested[strings.TrimSuffix(dir, "/")] = true
|
||||
}
|
||||
}
|
||||
inNested := func(dir string) bool {
|
||||
for d := dir; d != "." && d != ""; d = path.Dir(d) {
|
||||
if nested[d] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
isPackageDir := map[string]bool{}
|
||||
for p := range files {
|
||||
dir := path.Dir(p)
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
if !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") || ignoredDir(dir) || inNested(dir) {
|
||||
continue
|
||||
}
|
||||
isPackageDir[dir] = true
|
||||
}
|
||||
for _, dir := range slices.Sorted(maps.Keys(isPackageDir)) {
|
||||
var lines []string
|
||||
for p, object := range files {
|
||||
rel, ok := under(p, dir)
|
||||
if !ok || strings.HasSuffix(p, "_test.go") {
|
||||
continue
|
||||
}
|
||||
if sub := path.Dir(rel); sub != "." && cutBySub(dir, sub, isPackageDir, nested) {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, rel+" "+object)
|
||||
}
|
||||
sort.Strings(lines)
|
||||
s := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
||||
imp := module
|
||||
if dir != "" {
|
||||
imp = module + "/" + dir
|
||||
}
|
||||
packages = append(packages, delivery.PackageDigest{Package: imp, Digest: hex.EncodeToString(s[:])})
|
||||
}
|
||||
slices.SortFunc(packages, func(a, b delivery.PackageDigest) int { return strings.Compare(a.Package, b.Package) })
|
||||
return requires, packages, nil
|
||||
}
|
||||
|
||||
// ignoredDir is whether the go tool takes no package from dir: an element named testdata, or starting with _ or .
|
||||
func ignoredDir(dir string) bool {
|
||||
for _, e := range strings.Split(dir, "/") {
|
||||
if e == "testdata" || strings.HasPrefix(e, "_") || strings.HasPrefix(e, ".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// under is p relative to dir ("" for the module's root), and whether p is under it.
|
||||
func under(p, dir string) (string, bool) {
|
||||
if dir == "" {
|
||||
return p, true
|
||||
}
|
||||
return strings.CutPrefix(p, dir+"/")
|
||||
}
|
||||
|
||||
// cutBySub is whether a file in the sub-directory sub of the package dir lies under another package or module.
|
||||
func cutBySub(dir, sub string, isPackageDir, nested map[string]bool) bool {
|
||||
acc := dir
|
||||
for _, e := range strings.Split(sub, "/") {
|
||||
if acc == "" {
|
||||
acc = e
|
||||
} else {
|
||||
acc += "/" + e
|
||||
}
|
||||
if isPackageDir[acc] || nested[acc] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys(m)) }
|
||||
|
||||
// HeldVersionsOf is the version line's version of each mesh Go module (by module path) as the indexer has always read
|
||||
// it (novox/hq ADR 0573 §2), so a module with no reported import set keeps the fingerprint it had: the trunk merge
|
||||
// commit of the held module of the package's repository whose directory holds the package's directory (the deepest, a
|
||||
// module at the repository's root holding every directory). A package no held module holds, or whose module names no
|
||||
// commit or no directory, is in missing, sorted; versions is nil when packages is empty.
|
||||
func HeldVersionsOf(packages []string, modules map[string]delivery.Module) (versions delivery.PackageVersions, missing []string) {
|
||||
for _, pkg := range packages {
|
||||
repository, dir, ok := delivery.PackageRepository(pkg)
|
||||
commit, depth := "", -1
|
||||
if ok {
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !delivery.SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == delivery.RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
commit, depth = m.Commit, d
|
||||
}
|
||||
}
|
||||
}
|
||||
if commit == "" {
|
||||
missing = append(missing, pkg)
|
||||
continue
|
||||
}
|
||||
if versions == nil {
|
||||
versions = delivery.PackageVersions{}
|
||||
}
|
||||
versions[pkg] = commit
|
||||
}
|
||||
slices.Sort(missing)
|
||||
return versions, missing
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const sdk = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
func base() Inputs {
|
||||
return Inputs{Module: "netcheck", Tree: "t1", Bases: []string{"src2:b"}, Contexts: map[string]string{"c": "x"},
|
||||
Locks: map[string]string{"bin": "l1"}, Toolchains: map[string]delivery.Toolchain2{"bin": {Language: "go", Version: "1.26.0", Recipe: "7"}}}
|
||||
}
|
||||
|
||||
// The form moved out of mesh-sdk computes what mesh-sdk's SourceFingerprint2 and NamedTree computed for the same
|
||||
// inputs (golden values taken from mesh-sdk 3d87ddc), so the move itself rebuilds nothing.
|
||||
func TestTheMovedComputationGivesTheValuesItGaveInMeshSDK(t *testing.T) {
|
||||
in := base()
|
||||
in.Packages = map[string]string{sdk: "abc"}
|
||||
if fp, none := Of(in); fp != "src2:0f96be10d19712c94ea0ece8125a45cbb658ec06b705b28e747332cc94500e41" || none != "" {
|
||||
t.Fatalf("with a package version: %q %q", fp, none)
|
||||
}
|
||||
if fp, _ := Of(base()); fp != "src2:f1a869faf14a3e422feba672156b42b1a7af7e7a1097c11554299f9e37e58625" {
|
||||
t.Fatalf("without: %q", fp)
|
||||
}
|
||||
named, err := NamedTree(strings.Repeat("0", 64), map[string]string{"cmd": "t9", "go.mod": "b1"})
|
||||
if err != nil || named != "named:0290bf64fc93b1c10954797d548be30d936008b041c58261faad0c388a3d4c80" {
|
||||
t.Fatalf("named tree: %q %v", named, err)
|
||||
}
|
||||
in.Packages = map[string]string{sdk: ""}
|
||||
if fp, none := Of(in); fp != "" || !strings.HasPrefix(none, NoPackageVersion) {
|
||||
t.Fatalf("a package with no version: %q %q", fp, none)
|
||||
}
|
||||
}
|
||||
|
||||
func published(module, repo, dir, commit, requires string, digests map[string]string) delivery.PublishedPackages {
|
||||
p := delivery.PublishedPackages{Module: module, Repository: repo, Dir: dir, Commit: commit, Requires: requires}
|
||||
for _, pkg := range sortedKeys(digests) {
|
||||
p.Packages = append(p.Packages, delivery.PackageDigest{Package: pkg, Digest: digests[pkg]})
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// ADR 0593 §2, §3: the fingerprint takes the digest of each package a module imports, and of each mesh Go module's
|
||||
// requirements; a package it does not import moves nothing, an imported one or a sibling one moves it.
|
||||
func TestTheImportFormMovesOnlyWithWhatTheModuleImports(t *testing.T) {
|
||||
const sibling = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
imports := []string{sdk, sdk + "/announce", sibling}
|
||||
pubs := func(announce, buildstep, sib, requires string) map[string]delivery.PublishedPackages {
|
||||
return map[string]delivery.PublishedPackages{
|
||||
sdk: published(sdk, "novox/mesh-sdk", "go", "c1", requires, map[string]string{sdk: "r", sdk + "/announce": announce, sdk + "/buildstep": buildstep}),
|
||||
sibling: published(sibling, "novox/mesh-catalog", "modules/mesh-build-go/buildstep", "c9", "q", map[string]string{sibling: sib}),
|
||||
}
|
||||
}
|
||||
fpOf := func(p map[string]delivery.PublishedPackages) string {
|
||||
t.Helper()
|
||||
r := Resolve("novox/mesh-catalog", imports, p)
|
||||
if len(r.Missing) > 0 {
|
||||
t.Fatalf("missing %v", r.Missing)
|
||||
}
|
||||
in := base()
|
||||
in.Imports, in.Requires = r.Digests, r.Requires
|
||||
fp, none := Of(in)
|
||||
if fp == "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
first := fpOf(pubs("a1", "b1", "s1", "q1"))
|
||||
if got := fpOf(pubs("a1", "b2", "s1", "q1")); got != first {
|
||||
t.Fatal("a package the module does not import (buildstep) moved its fingerprint")
|
||||
}
|
||||
if got := fpOf(pubs("a2", "b1", "s1", "q1")); got == first {
|
||||
t.Fatal("an imported package (announce) changed and the fingerprint did not move")
|
||||
}
|
||||
if got := fpOf(pubs("a1", "b1", "s2", "q1")); got == first {
|
||||
t.Fatal("a sibling package of the module's own repository changed and the fingerprint did not move")
|
||||
}
|
||||
if got := fpOf(pubs("a1", "b1", "s1", "q2")); got == first {
|
||||
t.Fatal("mesh-sdk's requirements changed and the fingerprint did not move")
|
||||
}
|
||||
// Versions are pinned only for modules of another repository: a sibling is built from its directory.
|
||||
r := Resolve("novox/mesh-catalog", imports, pubs("a1", "b1", "s1", "q1"))
|
||||
if len(r.Versions) != 1 || r.Versions[sdk] != "c1" {
|
||||
t.Fatalf("versions %v", r.Versions)
|
||||
}
|
||||
if r := Resolve("novox/mesh-catalog", append(imports, sdk+"/gone"), pubs("a1", "b1", "s1", "q1")); len(r.Missing) != 1 {
|
||||
t.Fatalf("an import nobody publishes is missing: %v", r.Missing)
|
||||
}
|
||||
}
|
||||
|
||||
// A package's digest is its own files: a test file, a sub-package or a nested module changes nothing of it; a file
|
||||
// of it, or of a sub-directory that is no package (testdata), changes only it.
|
||||
func TestAPackageDigestIsItsOwnFiles(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"go.mod": "m1", "go.sum": "s1", "stdio.go": "a", "stdio_test.go": "t",
|
||||
"announce/announce.go": "b", "announce/testdata/x.json": "d",
|
||||
"buildstep/step.go": "c", "buildstep/conformance/c.go": "e",
|
||||
"tools/go.mod": "nm", "tools/t.go": "f", "internal/x/x.go": "g", "_old/o.go": "h",
|
||||
}
|
||||
requires, pkgs, err := Packages(sdk, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests := map[string]string{}
|
||||
for _, p := range pkgs {
|
||||
digests[p.Package] = p.Digest
|
||||
}
|
||||
want := []string{sdk, sdk + "/announce", sdk + "/buildstep", sdk + "/buildstep/conformance", sdk + "/internal/x"}
|
||||
if got := sortedKeys(digests); strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("packages %v, want %v", got, want)
|
||||
}
|
||||
change := func(path, object string) map[string]string {
|
||||
_, ps, err := Packages(sdk, with(files, path, object))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, p := range ps {
|
||||
out[p.Package] = p.Digest
|
||||
}
|
||||
return out
|
||||
}
|
||||
moved := func(after map[string]string) (out []string) {
|
||||
for p, d := range after {
|
||||
if digests[p] != d {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
for path, want := range map[string]string{
|
||||
"stdio_test.go": "",
|
||||
"buildstep/conformance/c.go": sdk + "/buildstep/conformance",
|
||||
"announce/testdata/x.json": sdk + "/announce",
|
||||
"stdio.go": sdk,
|
||||
"tools/t.go": "",
|
||||
} {
|
||||
got := moved(change(path, "zz"))
|
||||
if (want == "" && len(got) != 0) || (want != "" && (len(got) != 1 || got[0] != want)) {
|
||||
t.Errorf("changing %s moved %v, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
if r2, _, _ := Packages(sdk, with(files, "go.sum", "s2")); r2 == requires {
|
||||
t.Error("a changed go.sum left the requirements' digest")
|
||||
}
|
||||
if _, _, err := Packages(sdk, map[string]string{"a.go": "x"}); err == nil {
|
||||
t.Error("a listing with no go.mod was taken as a module")
|
||||
}
|
||||
}
|
||||
|
||||
func with(m map[string]string, k, v string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for a, b := range m {
|
||||
out[a] = b
|
||||
}
|
||||
out[k] = v
|
||||
return out
|
||||
}
|
||||
@@ -58,6 +58,16 @@ type Indexer struct {
|
||||
JudgeID string
|
||||
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
|
||||
SayRefused func(ctx context.Context, r delivery.ManifestRefused)
|
||||
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
|
||||
// and every fingerprint takes the version line.
|
||||
Packages Store
|
||||
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
|
||||
// with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from.
|
||||
// Nil publishes no package.
|
||||
Files func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error)
|
||||
// GoSum reads a module directory's go.sum at a commit (gitea.gitea_get_file): the mesh Go modules a module with no
|
||||
// reported import set takes the version line of. Nil reads none: such a module keeps what it held.
|
||||
GoSum func(ctx context.Context, repository, sha, dir string) ([]byte, error)
|
||||
// Now is the clock of a taken value; nil is time.Now.
|
||||
Now func() time.Time
|
||||
Builds *Builds
|
||||
@@ -80,19 +90,20 @@ type Indexer struct {
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
previews map[string]delivery.Preview // by previews key (previews.go)
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// form is a module's fingerprint, or why it has none, and the version of each mesh Go module of another repository
|
||||
// its build is pinned to.
|
||||
type form struct {
|
||||
fp, none string
|
||||
versions delivery.PackageVersions
|
||||
}
|
||||
|
||||
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
|
||||
// form they call for (novox/hq ADR 0593 §2, §4):
|
||||
//
|
||||
// - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is
|
||||
// published (the packages state): the content digest of each imported package and the requirements' digest of each
|
||||
// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only).
|
||||
// - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs
|
||||
// the build that reported the imports was built from. While the import form computes to it, nothing the build was
|
||||
// built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions.
|
||||
// Otherwise the import form is written, and ImportForm moves with it.
|
||||
// - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module its
|
||||
// go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), so its value is the one it had.
|
||||
//
|
||||
// It sets kept.ImportForm when it writes the import form.
|
||||
func formOf(module, repository string, kept *delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2, prev delivery.Module, held map[string]delivery.Module, published map[string]delivery.PublishedPackages, logf func(string, ...any)) form {
|
||||
in := inputsFrom(module, *kept, toolchains)
|
||||
if len(kept.Imports) > 0 {
|
||||
r := fingerprint.Resolve(repository, kept.Imports, published)
|
||||
if len(r.Missing) == 0 {
|
||||
in.Imports, in.Requires = r.Digests, r.Requires
|
||||
fp, none := fingerprint.Of(in)
|
||||
switch {
|
||||
case fp == "":
|
||||
return form{none: none}
|
||||
case kept.ImportForm == fp && prev.FP2 != "":
|
||||
return form{fp: prev.FP2, versions: prev.Packages} // the build that reported the imports stands
|
||||
}
|
||||
kept.ImportForm = fp
|
||||
return form{fp: fp, versions: r.Versions}
|
||||
}
|
||||
if logf != nil {
|
||||
logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", "))
|
||||
}
|
||||
}
|
||||
versions, unpublished := fingerprint.HeldVersionsOf(kept.Packages, held)
|
||||
if len(unpublished) > 0 && logf != nil {
|
||||
logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", "))
|
||||
}
|
||||
if len(kept.Packages) > 0 {
|
||||
in.Packages = map[string]string{}
|
||||
for _, p := range kept.Packages {
|
||||
in.Packages[p] = versions[p]
|
||||
}
|
||||
}
|
||||
fp, none := fingerprint.Of(in)
|
||||
if fp == "" {
|
||||
return form{none: none}
|
||||
}
|
||||
return form{fp: fp, versions: versions}
|
||||
}
|
||||
|
||||
// inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of
|
||||
// either form yet.
|
||||
func inputsFrom(module string, k delivery.KeptInputs2, toolchains map[string]delivery.Toolchain2) fingerprint.Inputs {
|
||||
return fingerprint.Inputs{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts,
|
||||
Locks: k.Locks, Toolchains: toolchains}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// ApplyBuildEnd keeps the imports a done build reports at its end (delivery.BuildLogEntry.Imports; novox/hq ADR 0593
|
||||
// §2) on the module it built, when the build's fingerprint is the one the module holds: the import set of the module's
|
||||
// last trunk build. With them it keeps ImportForm, the import-form fingerprint of exactly what the build was built from:
|
||||
// the module's kept inputs and the toolchains its fingerprint was computed with, and each imported package's digest at
|
||||
// the version the build took it at (another repository's at the version the module's fingerprint pins, its own
|
||||
// repository's at the commit built). A recompute that gives that value again keeps the fingerprint (§4); one that
|
||||
// differs writes the import form. The fingerprint itself is not changed here.
|
||||
//
|
||||
// An entry that is no done build's end, reports no imports, or is of a fingerprint the module no longer holds is left.
|
||||
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
|
||||
// resolves them writes the import form, one rebuild too many, never a stale module.
|
||||
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
|
||||
e, err := delivery.DecodeBuildLogEntry(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.ModulesKey(e.Module)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
raw, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
cur, err := delivery.DecodeModule(raw)
|
||||
if err != nil || cur.FP2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
|
||||
return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
|
||||
}
|
||||
if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" {
|
||||
return nil
|
||||
}
|
||||
_, m, err := ix.manifestRaw(ctx, cur.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: its manifest: %w", key, err)
|
||||
}
|
||||
toolchains := map[string]delivery.Toolchain2{}
|
||||
for artifact, language := range languagesOf(m) {
|
||||
used, ok := cur.Toolchains[language]
|
||||
if !ok {
|
||||
ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language)
|
||||
return nil
|
||||
}
|
||||
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
kept := *cur.Inputs2
|
||||
kept.Imports, kept.ImportForm = slices.Clone(e.Imports), ""
|
||||
if built, err := ix.publishedAsBuilt(ctx, cur, e); err != nil {
|
||||
ix.logf("%s: the packages its build imported cannot be read as built (%v): its imports are kept, and its next recompute writes the import form", e.Module, err)
|
||||
} else if r := fingerprint.Resolve(cur.Repository, e.Imports, built); len(r.Missing) > 0 {
|
||||
ix.logf("%s: no published package holds %v: its imports are kept, and the recompute that resolves them writes the import form", e.Module, r.Missing)
|
||||
} else {
|
||||
in := inputsFrom(e.Module, kept, toolchains)
|
||||
in.Imports, in.Requires = r.Digests, r.Requires
|
||||
if fp, _ := fingerprint.Of(in); fp != "" {
|
||||
kept.ImportForm = fp
|
||||
}
|
||||
}
|
||||
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
now, err := delivery.DecodeModule(value)
|
||||
if err != nil || now.FP2 != e.FP || now.Manifest != cur.Manifest || now.Inputs2 == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
in := *now.Inputs2
|
||||
in.Imports, in.ImportForm = kept.Imports, kept.ImportForm
|
||||
now.Inputs2 = &in
|
||||
data, err := delivery.EncodeModule(now)
|
||||
return data, false, err
|
||||
})
|
||||
}
|
||||
|
||||
// publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took
|
||||
// it: a module of the built module's own repository at the commit built, one of another repository at the version the
|
||||
// module's fingerprint pins (cur.Packages), else its latest published version. A module whose published commit is the
|
||||
// one built is taken from the packages state; any other is read from the forge's listing at that commit.
|
||||
func (ix *Indexer) publishedAsBuilt(ctx context.Context, cur delivery.Module, e delivery.BuildLogEntry) (map[string]delivery.PublishedPackages, error) {
|
||||
published := ix.publishedAll(ctx)
|
||||
if r := fingerprint.Resolve(cur.Repository, e.Imports, published); len(r.Missing) > 0 {
|
||||
if err := ix.FillPackages(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
published = ix.publishedAll(ctx)
|
||||
}
|
||||
type at struct{ repository, commit string }
|
||||
reread := map[at][]string{}
|
||||
for _, imp := range e.Imports {
|
||||
r := fingerprint.Resolve(cur.Repository, []string{imp}, published)
|
||||
for mod := range r.Requires {
|
||||
p := published[mod]
|
||||
commit := p.Commit
|
||||
switch {
|
||||
case delivery.SameRepository(p.Repository, cur.Repository) && e.Commit != "":
|
||||
commit = e.Commit
|
||||
case cur.Packages[mod] != "":
|
||||
commit = cur.Packages[mod]
|
||||
}
|
||||
if commit != p.Commit {
|
||||
k := at{p.Repository, commit}
|
||||
if !slices.Contains(reread[k], mod) {
|
||||
reread[k] = append(reread[k], mod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out := maps.Clone(published)
|
||||
for k, mods := range reread {
|
||||
got, err := ix.packagesAt(ctx, k.repository, k.commit, mods, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maps.Copy(out, got)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const meshSDK = "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
// Full commits of mesh-sdk and of the catalogue.
|
||||
var (
|
||||
c1 = strings.Repeat("1", 40)
|
||||
c2 = strings.Repeat("2", 40)
|
||||
c3 = strings.Repeat("3", 40)
|
||||
cat1 = strings.Repeat("a", 40)
|
||||
)
|
||||
|
||||
// sdkFiles is a listing of mesh-sdk's go directory (gitea_commit_trees with files): announce and buildstep at the
|
||||
// given blobs.
|
||||
func sdkFiles(announce, buildstep string) map[string]string {
|
||||
return map[string]string{"go.mod": "m1", "go.sum": "s1", "stdio.go": "r1", "announce/announce.go": announce, "buildstep/step.go": buildstep}
|
||||
}
|
||||
|
||||
// forge answers Files from the listing of each commit of mesh-sdk.
|
||||
type forge map[string]map[string]string
|
||||
|
||||
func (f forge) files(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, d := range dirs {
|
||||
if repository != "novox/mesh-sdk" || d != "go" || f[sha] == nil {
|
||||
return nil, &missingDir{d}
|
||||
}
|
||||
out[d] = f[sha]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type missingDir struct{ dir string }
|
||||
|
||||
func (m *missingDir) Error() string { return "no directory " + m.dir }
|
||||
|
||||
// ADR 0593 §2-§4 (D1, D2, D3): a build's imports are kept with no fingerprint moving; a mesh-sdk merge that changes
|
||||
// a package the module does not import leaves it; one that changes an imported package moves it to the import form;
|
||||
// and the packages state holds one key with a digest per package.
|
||||
func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
listings := forge{c1: sdkFiles("a1", "b1"), c2: sdkFiles("a1", "b2"), c3: sdkFiles("a2", "b2")}
|
||||
ix.Packages, ix.Files = newMem(), listings.files
|
||||
// avahi's go.sum names mesh-sdk (the version line), as every Go module's does; mesh-sdk's module is held at c1.
|
||||
setPackages(t, mods, "avahi", []string{meshSDK})
|
||||
sdkModule, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Path: ".", Commit: c1, None2: "a TypeScript package"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mods.put("mesh-sdk", sdkModule)
|
||||
held, _ := ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := ix.publishedAll(ctx)[meshSDK]
|
||||
if pub.Commit != c1 || len(pub.Packages) != 3 || pub.Requires == "" {
|
||||
t.Fatalf("packages.%s = %+v, want commit c1 and a digest per package (go, announce, buildstep)", meshSDK, pub)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := moduleOf(t, mods, "avahi")
|
||||
if before.FP2 == "" {
|
||||
t.Fatalf("no fingerprint to start from: %q", before.None2)
|
||||
}
|
||||
|
||||
// The build of that fingerprint ends, reporting its imports: kept, and the fingerprint does not move.
|
||||
end := buildEnd(t, "avahi", before.FP2, meshSDK, meshSDK+"/announce")
|
||||
if err := ix.ApplyBuildEnd(ctx, end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := moduleOf(t, mods, "avahi")
|
||||
if after.FP2 != before.FP2 || strings.Join(after.Inputs2.Imports, ",") != meshSDK+","+meshSDK+"/announce" || after.Inputs2.ImportForm == "" {
|
||||
t.Fatalf("after the build's imports: fp2 %q (was %q), imports %v, import form %q; want the same fingerprint and the imports kept",
|
||||
after.FP2, before.FP2, after.Inputs2.Imports, after.Inputs2.ImportForm)
|
||||
}
|
||||
|
||||
// mesh-sdk merges a change of buildstep only: avahi does not import it.
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.FP2 != before.FP2 {
|
||||
t.Fatalf("a mesh-sdk merge changing only buildstep moved avahi's fingerprint to %q", m.FP2)
|
||||
}
|
||||
|
||||
// mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version.
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _ := moduleOf(t, mods, "avahi")
|
||||
if moved.FP2 == before.FP2 || moved.FP2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
|
||||
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3",
|
||||
moved.FP2, before.FP2, moved.Inputs2.ImportForm, moved.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept.
|
||||
func TestABuildOfAnOlderFingerprintKeepsNoImports(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
if err := ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || len(m.Inputs2.Imports) != 0 {
|
||||
t.Fatalf("imports %v kept from a build of another fingerprint", m.Inputs2)
|
||||
}
|
||||
}
|
||||
|
||||
func buildEnd(t *testing.T, module, fp string, imports ...string) []byte {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: fp,
|
||||
Repository: catalogueRepo, Directory: "modules/" + module, Commit: cat1, Ref: "main", Node: "n1", At: time.Now(),
|
||||
Outcome: delivery.BuildDone, Imports: imports})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// setPackages writes a module's version-line packages, as a merge reading its go.sum keeps them.
|
||||
func setPackages(t *testing.T, mods *memStore, module string, packages []string) {
|
||||
t.Helper()
|
||||
m, _ := moduleOf(t, mods, module)
|
||||
in := *m.Inputs2
|
||||
in.Packages = packages
|
||||
m.Inputs2 = &in
|
||||
raw, err := delivery.EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mods.put(module, raw)
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
@@ -55,7 +56,7 @@ type baseFP func(module string) (fp, none string)
|
||||
// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
|
||||
// resolves packages when it is built (wantsPackages, :1017) only when its language's lock file exists: it then has
|
||||
// an entry in Locks with that file's hash, and an artifact whose lock file the merge did not carry has no entry (a
|
||||
// dependency-free or vendored bundle resolves nothing, as mesh-sdk SourceInputs2 says of an artifact not in Locks;
|
||||
// dependency-free or vendored bundle resolves nothing, as fingerprint.Inputs says of an artifact not in Locks;
|
||||
// novox/hq ADR 0469 clause 5, issue 1459). A bundle that needs a lock and lacks one is refused where it is built, not
|
||||
// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve
|
||||
// nothing and have no toolchain. A base is a build.on entry that names a module.
|
||||
@@ -65,7 +66,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo
|
||||
|
||||
// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery
|
||||
// Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its
|
||||
// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs
|
||||
// manifest, by the indexer's one function (fingerprint.NamedTree), and keeps their objects, so a computation from kept inputs
|
||||
// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it
|
||||
// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given.
|
||||
//
|
||||
@@ -104,7 +105,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, p
|
||||
if len(missing) > 0 {
|
||||
return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths}
|
||||
}
|
||||
if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
|
||||
if tree, err = fingerprint.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
|
||||
return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
@@ -97,9 +98,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool,
|
||||
if next.Manifest == "" {
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
} else {
|
||||
packages := delivery.OtherPackages(f.Packages[dir], f.Repository) // its own repository's come with its tree
|
||||
packages := ix.goSumPackages(ctx, f, dir, cur, has)
|
||||
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
|
||||
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
|
||||
if in.Kept != nil && has && cur.Inputs2 != nil {
|
||||
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
|
||||
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
|
||||
}
|
||||
if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
|
||||
// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
|
||||
// that reads it judges the module again at start with no merge (issue 1923).
|
||||
@@ -118,6 +123,13 @@ func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool,
|
||||
return w, err
|
||||
}
|
||||
}
|
||||
// The packages this merge publishes come first, so the modules importing a changed one move in this recompute (ADR
|
||||
// 0593 §3). A forge that cannot list them leaves the state as it was, said: the importers move at the next merge.
|
||||
reach := maps.Clone(held)
|
||||
maps.Copy(reach, overlay)
|
||||
if err := ix.publish(ctx, f.Repository, f.Merge, reach, false); err != nil {
|
||||
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
|
||||
}
|
||||
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
|
||||
return w, ix.recomputeWith(ctx, overlay, w)
|
||||
}
|
||||
@@ -320,3 +332,39 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
|
||||
in.Packages = slices.Clone(packages)
|
||||
return in
|
||||
}
|
||||
|
||||
// goSumPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names (the version
|
||||
// line's packages, ADR 0573 §2), read by the indexer itself (the bridge reads none since ADR 0593 §2): those the message
|
||||
// carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read from the
|
||||
// forge (GoSum). A directory with no go.sum has none.
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
if carried := f.Packages[dir]; len(carried) > 0 {
|
||||
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
|
||||
}
|
||||
hash := f.Locks[dir]["go.sum"]
|
||||
if hash == "" {
|
||||
return nil
|
||||
}
|
||||
if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) {
|
||||
for _, h := range cur.Inputs2.Locks {
|
||||
if h == hash {
|
||||
return cur.Inputs2.Packages
|
||||
}
|
||||
}
|
||||
}
|
||||
if ix.GoSum == nil {
|
||||
if has && cur.Inputs2 != nil {
|
||||
return cur.Inputs2.Packages
|
||||
}
|
||||
return nil
|
||||
}
|
||||
raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
|
||||
if has && cur.Inputs2 != nil {
|
||||
return cur.Inputs2.Packages
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// The published packages (novox/hq ADR 0593 §2): the indexer alone reads, from the forge's listing, the content digest
|
||||
// of each package of every mesh Go module a held module reaches, and writes the module's latest published version and
|
||||
// digests in its state `packages` (delivery.PackagesBucket), one key per Go module. The builder, the checker and the
|
||||
// build modules read it there; nobody else computes a digest or a version.
|
||||
//
|
||||
// A Go module is reached when a held module's go.sum names it (KeptInputs2.Packages) or holds an import a build
|
||||
// reported (KeptInputs2.Imports). Its key is written at each merge of its repository, before the merge's recompute, so
|
||||
// a merge that changes a package moves its importers in the same recompute (§3); and once at the start, for each
|
||||
// reached module the state does not hold yet (FillPackages).
|
||||
|
||||
// publishedAll is every value of the packages state, by Go module path; empty with no state or one that does not answer.
|
||||
func (ix *Indexer) publishedAll(ctx context.Context) map[string]delivery.PublishedPackages {
|
||||
out := map[string]delivery.PublishedPackages{}
|
||||
if ix.Packages == nil {
|
||||
return out
|
||||
}
|
||||
keys, err := ix.Packages.Keys(ctx)
|
||||
if err != nil {
|
||||
ix.logf("the packages state cannot be listed, so every fingerprint takes the version line: %v", err)
|
||||
return out
|
||||
}
|
||||
for _, key := range keys {
|
||||
value, _, found, err := ix.Packages.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePublishedPackages(value)
|
||||
if err != nil {
|
||||
ix.logf("packages.%s holds what is not a published module, left out: %v", key, err)
|
||||
continue
|
||||
}
|
||||
out[p.Module] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reached are, by repository, the mesh Go module paths the held modules' go.sum names and the imports their builds
|
||||
// reported.
|
||||
func reached(held map[string]delivery.Module) (modules, imports map[string][]string) {
|
||||
modules, imports = map[string][]string{}, map[string][]string{}
|
||||
add := func(to map[string][]string, p string) {
|
||||
repo, _, ok := delivery.PackageRepository(p)
|
||||
if ok && !slices.Contains(to[repo], p) {
|
||||
to[repo] = append(to[repo], p)
|
||||
}
|
||||
}
|
||||
for _, m := range held {
|
||||
if m.Inputs2 == nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range m.Inputs2.Packages {
|
||||
add(modules, p)
|
||||
}
|
||||
for _, p := range m.Inputs2.Imports {
|
||||
add(imports, p)
|
||||
}
|
||||
}
|
||||
for _, by := range []map[string][]string{modules, imports} {
|
||||
for r := range by {
|
||||
slices.Sort(by[r])
|
||||
}
|
||||
}
|
||||
return modules, imports
|
||||
}
|
||||
|
||||
// packagesAt are the Go modules of repository at commit that hold modules (module paths) or imports (import paths),
|
||||
// computed from the forge's listing (Files), by module path. An import is held by a known module when one holds its
|
||||
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
|
||||
// directory by directory). A module the listing has no go.mod for is said and left out.
|
||||
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]delivery.PublishedPackages, error) {
|
||||
if ix.Files == nil || commit == "" {
|
||||
return nil, nil
|
||||
}
|
||||
dirOf := map[string]string{} // module path to its directory in the repository
|
||||
for _, m := range modules {
|
||||
if _, dir, ok := delivery.PackageRepository(m); ok {
|
||||
dirOf[m] = dir
|
||||
}
|
||||
}
|
||||
holds := func(imp string) bool {
|
||||
for m := range dirOf {
|
||||
if imp == m || strings.HasPrefix(imp, m+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
listings := map[string]map[string]string{} // by directory
|
||||
list := func(dirs []string) error {
|
||||
var ask []string
|
||||
for _, d := range dirs {
|
||||
if _, done := listings[d]; !done {
|
||||
ask = append(ask, d)
|
||||
}
|
||||
}
|
||||
if len(ask) == 0 {
|
||||
return nil
|
||||
}
|
||||
got, err := ix.Files(ctx, repository, commit, ask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err)
|
||||
}
|
||||
for _, d := range ask {
|
||||
listings[d] = got[d]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, imp := range imports {
|
||||
if holds(imp) {
|
||||
continue
|
||||
}
|
||||
_, dir, _ := delivery.PackageRepository(imp)
|
||||
for d := dir; ; d = parent(d) {
|
||||
if err := list([]string{d}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := listings[d]["go.mod"]; ok {
|
||||
m := strings.TrimSuffix(delivery.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
|
||||
dirOf[m] = d
|
||||
break
|
||||
}
|
||||
if d == "" {
|
||||
ix.logf("%s at %s: no go.mod holds the import %s, so it has no digest", repository, commit, imp)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := list(slices.Sorted(maps.Values(dirOf))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]delivery.PublishedPackages{}
|
||||
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
|
||||
dir := dirOf[m]
|
||||
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: %v; it is not published", repository, commit, err)
|
||||
continue
|
||||
}
|
||||
d := dir
|
||||
if d == "" {
|
||||
d = delivery.RootPath
|
||||
}
|
||||
out[m] = delivery.PublishedPackages{Module: m, Repository: delivery.RepositoryName(repository), Dir: d, Commit: commit,
|
||||
Requires: requires, Packages: pkgs}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parent is the directory above d, "" for the repository's root.
|
||||
func parent(d string) string {
|
||||
if d == "" {
|
||||
return ""
|
||||
}
|
||||
p := path.Dir(d)
|
||||
if p == "." {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
|
||||
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
|
||||
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]delivery.Module, onlyMissing bool) error {
|
||||
if ix.Packages == nil || ix.Files == nil {
|
||||
return nil
|
||||
}
|
||||
modules, imports := reached(held)
|
||||
repo := delivery.RepositoryName(repository)
|
||||
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
|
||||
return nil
|
||||
}
|
||||
have := ix.publishedAll(ctx)
|
||||
if onlyMissing {
|
||||
// Imports a held module already resolves are not asked again.
|
||||
var ms, is []string
|
||||
for _, m := range modules[repo] {
|
||||
if _, ok := have[m]; !ok {
|
||||
ms = append(ms, m)
|
||||
}
|
||||
}
|
||||
for _, imp := range imports[repo] {
|
||||
if len(fingerprint.Resolve(repo, []string{imp}, have).Missing) > 0 {
|
||||
is = append(is, imp)
|
||||
}
|
||||
}
|
||||
if len(ms) == 0 && len(is) == 0 {
|
||||
return nil
|
||||
}
|
||||
modules[repo], imports[repo] = ms, is
|
||||
}
|
||||
// An import a published module of the repository already holds is resolved by its module's directory.
|
||||
for m, p := range have {
|
||||
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(modules[repo], m) && !onlyMissing {
|
||||
modules[repo] = append(modules[repo], m)
|
||||
}
|
||||
}
|
||||
got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range slices.Sorted(maps.Keys(got)) {
|
||||
next := got[m]
|
||||
key, err := delivery.PackagesKey(m)
|
||||
if err != nil {
|
||||
ix.logf("%s: %v", m, err)
|
||||
continue
|
||||
}
|
||||
err = write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodePublishedPackages(value); err == nil && cur.Commit == next.Commit &&
|
||||
cur.Requires == next.Requires && reflect.DeepEqual(cur.Packages, next.Packages) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
next.At = ix.now().UTC()
|
||||
data, err := delivery.EncodePublishedPackages(next)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("packages.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
|
||||
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
|
||||
// A repository with no taken merge is left for its next merge.
|
||||
func (ix *Indexer) FillPackages(ctx context.Context) error {
|
||||
if ix.Packages == nil || ix.Files == nil || ix.Modules == nil || ix.Taken == nil {
|
||||
return nil
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, imports := reached(held)
|
||||
repos := map[string]bool{}
|
||||
for r := range modules {
|
||||
repos[r] = true
|
||||
}
|
||||
for r := range imports {
|
||||
repos[r] = true
|
||||
}
|
||||
for _, repo := range slices.Sorted(maps.Keys(repos)) {
|
||||
key, err := delivery.RepositoryKey(repo)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Taken.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
t, err := delivery.DecodeTaken(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
|
||||
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidatePublished is the packages state as a candidate builds against it (novox/hq ADR 0573 §4): each Go module a
|
||||
// member's repository publishes at the member's head (members, by repository), every other at its latest published
|
||||
// version. A head's modules are read from the forge once and kept while the head is a member somewhere; one the forge
|
||||
// cannot list is said and left at its published version.
|
||||
func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]string, held map[string]delivery.Module) map[string]delivery.PublishedPackages {
|
||||
out := ix.publishedAll(ctx)
|
||||
if len(members) == 0 {
|
||||
return out
|
||||
}
|
||||
modules, imports := reached(held)
|
||||
for _, repo := range slices.Sorted(maps.Keys(members)) {
|
||||
head := members[repo]
|
||||
if len(modules[repo]) == 0 && len(imports[repo]) == 0 {
|
||||
continue
|
||||
}
|
||||
at := repo + "@" + head
|
||||
ix.mu.Lock()
|
||||
got, ok := ix.atHeads[at]
|
||||
ix.mu.Unlock()
|
||||
if !ok {
|
||||
var ms []string
|
||||
ms = append(ms, modules[repo]...)
|
||||
for m, p := range out {
|
||||
if delivery.SameRepository(p.Repository, repo) && !slices.Contains(ms, m) {
|
||||
ms = append(ms, m)
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil {
|
||||
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
|
||||
continue
|
||||
}
|
||||
ix.mu.Lock()
|
||||
if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
|
||||
ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
|
||||
}
|
||||
ix.atHeads[at] = got
|
||||
ix.mu.Unlock()
|
||||
}
|
||||
maps.Copy(out, got)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again.
|
||||
const maxHeadsKept = 64
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull
|
||||
@@ -254,7 +256,15 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
|
||||
out.None2 = NonePreviewNoManifest
|
||||
return out
|
||||
}
|
||||
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, delivery.OtherPackages(pm.Packages, pv.Repository), ix.baseFP(ctx))
|
||||
// The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer
|
||||
// computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build
|
||||
// module still names them.
|
||||
trunk := held[m]
|
||||
packages := fingerprint.OtherPackages(pm.Packages, pv.Repository)
|
||||
if len(pm.Packages) == 0 && trunk.Inputs2 != nil {
|
||||
packages = trunk.Inputs2.Packages
|
||||
}
|
||||
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
|
||||
if in.Kept == nil {
|
||||
out.None2 = in.None2
|
||||
return out
|
||||
@@ -274,8 +284,29 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
|
||||
toolchains[artifact] = t.tc
|
||||
}
|
||||
inputs := *in.Kept // no builder input (ADR 0530 §1)
|
||||
versions := candidatePackages(inputs.Packages, held, members)
|
||||
fp, none := delivery.SourceFingerprint2(inputs.WithPackages(m, toolchains, versions))
|
||||
var fp, none string
|
||||
var versions delivery.PackageVersions
|
||||
if trunk.Inputs2 != nil && len(trunk.Inputs2.Imports) > 0 {
|
||||
inputs.Imports, inputs.ImportForm = trunk.Inputs2.Imports, trunk.Inputs2.ImportForm
|
||||
}
|
||||
// The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk
|
||||
// computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed.
|
||||
published := ix.candidatePublished(ctx, members, held)
|
||||
if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 {
|
||||
f := formOf(m, pv.Repository, &inputs, toolchains, trunk, held, published, nil)
|
||||
fp, none, versions = f.fp, f.none, f.versions
|
||||
} else {
|
||||
inputs.Imports, inputs.ImportForm = nil, ""
|
||||
versions = candidatePackages(inputs.Packages, held, members)
|
||||
in := inputsFrom(m, inputs, toolchains)
|
||||
if len(inputs.Packages) > 0 {
|
||||
in.Packages = map[string]string{}
|
||||
for _, p := range inputs.Packages {
|
||||
in.Packages[p] = versions[p]
|
||||
}
|
||||
}
|
||||
fp, none = fingerprint.Of(in)
|
||||
}
|
||||
if fp == "" {
|
||||
out.None2 = none
|
||||
return out
|
||||
@@ -291,7 +322,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
|
||||
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
|
||||
// version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none.
|
||||
func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions {
|
||||
versions, _ := delivery.PackageVersionsOf(packages, held)
|
||||
versions, _ := fingerprint.HeldVersionsOf(packages, held)
|
||||
for _, p := range packages {
|
||||
repository, _, _ := delivery.PackageRepository(p)
|
||||
if head := members[repository]; head != "" {
|
||||
|
||||
@@ -199,6 +199,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
published := ix.publishedAll(ctx)
|
||||
type result struct {
|
||||
fp, none string
|
||||
tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
|
||||
@@ -254,12 +255,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
}
|
||||
toolchains[artifact] = t.tc
|
||||
}
|
||||
// The latest published version of each mesh package the module uses: the trunk merge of the module that holds it,
|
||||
// as this recompute holds it, so a package's merge moves every dependent in the same recompute (ADR 0573 §2).
|
||||
versions, unpublished := delivery.PackageVersionsOf(kept.Packages, held)
|
||||
if len(unpublished) > 0 {
|
||||
ix.logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", "))
|
||||
}
|
||||
var r result
|
||||
switch {
|
||||
case k.contexts:
|
||||
@@ -268,13 +263,14 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
if fp, ok := stillComputed(module, kept, k, toolchains, versions); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: versions}
|
||||
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, published); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
|
||||
}
|
||||
default:
|
||||
r.fp, r.none = delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
|
||||
f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, published, ix.logf)
|
||||
r.fp, r.none = f.fp, f.none
|
||||
if r.fp != "" {
|
||||
r.tcs, r.pkgs = toolchainsUsed(toolchains), versions
|
||||
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
|
||||
}
|
||||
}
|
||||
done[module] = r
|
||||
@@ -421,6 +417,7 @@ func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur deliv
|
||||
if in.Kept == nil {
|
||||
return cur, false
|
||||
}
|
||||
in.Kept.Imports, in.Kept.ImportForm = cur.Inputs2.Imports, cur.Inputs2.ImportForm
|
||||
cur.Inputs2, cur.None2, cur.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
|
||||
return cur, true
|
||||
}
|
||||
@@ -442,11 +439,12 @@ func hasContext(m catalogue.Manifest) bool {
|
||||
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
|
||||
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
|
||||
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
|
||||
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
|
||||
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, versions delivery.PackageVersions) (string, bool) {
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
|
||||
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
|
||||
// module has none until they agree.
|
||||
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, held map[string]delivery.Module, published map[string]delivery.PublishedPackages) (string, delivery.PackageVersions, bool) {
|
||||
if k.held.FP2 == "" {
|
||||
return "", false
|
||||
return "", nil, false
|
||||
}
|
||||
toolchains := map[string]delivery.Toolchain2{}
|
||||
for artifact, language := range k.languages {
|
||||
@@ -456,12 +454,12 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma
|
||||
}
|
||||
used, ok := k.held.Toolchains[language]
|
||||
if !ok {
|
||||
return "", false
|
||||
return "", nil, false
|
||||
}
|
||||
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
fp, none := delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
|
||||
return fp, none == "" && fp == k.held.FP2
|
||||
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, nil)
|
||||
return f.fp, f.versions, f.none == "" && f.fp == k.held.FP2
|
||||
}
|
||||
|
||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
@@ -87,7 +89,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("emits = %v, want manifest-refused alone", m.Emits)
|
||||
}
|
||||
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
{ "name": "config-candidates", "history": 5 },
|
||||
{ "name": "taken", "history": 1 },
|
||||
{ "name": "takeins", "history": 1 },
|
||||
{ "name": "verdicts", "history": 5 }
|
||||
{ "name": "verdicts", "history": 5 },
|
||||
{ "name": "packages", "history": 5 }
|
||||
],
|
||||
"emits": [
|
||||
"manifest-refused"
|
||||
|
||||
Reference in New Issue
Block a user