mesh-delivery-indexer: a module's mesh packages are what its go.mod requires
At main's revision 21, 64 of 82 Go modules carried no package edge: the version line read the mesh packages go.sum names, and go.sum names none required at the zero version (ADR 0573 §2). The take-in now reads them from the go.mod's require lines, at any version, with no build needed; the go.sum is read only for the requirements state. The sibling rule (replaces by a directory) stands. Builds: design 47 H H-A-157, H-A-136 (sibling rule), G §47 (kind 1)
This commit is contained in:
@@ -182,9 +182,9 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// readVia reads a repository at a commit as the forge bridge does for a merged message: the seat git's trees, then the
|
||||
// module.json and lock files of the one directory dir by its file (a repository can hold hundreds). A manifest the forge
|
||||
// does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does
|
||||
// not have is left out.
|
||||
// module.json and lock files of the one directory dir by its file (a repository can hold hundreds), and the go.mod beside a
|
||||
// go.sum for its mesh packages (Packages). A manifest the forge does not have is an error; a directory with no module.json in
|
||||
// the trees is answered with no manifest; a lock file it does not have is left out.
|
||||
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
a, err := treesOf(call, repository, git.TreesInput{Commit: sha})
|
||||
@@ -222,13 +222,20 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
out.Locks[dir] = map[string]string{}
|
||||
}
|
||||
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||
if name == "go.sum" {
|
||||
if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 {
|
||||
if out.Packages == nil {
|
||||
out.Packages = map[string][]string{}
|
||||
}
|
||||
out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2)
|
||||
if name != "go.sum" {
|
||||
continue
|
||||
}
|
||||
// A Go module's mesh packages are what its go.mod requires, at any version (design 47 H-A-157): its go.sum
|
||||
// names none required at the zero version (ADR 0573 §2).
|
||||
mod, absent, err := file(dir, "go.mod")
|
||||
if err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "go.mod"), err)
|
||||
}
|
||||
if p := fingerprint.MeshRequiresOf([]byte(mod)); !absent && len(p) > 0 {
|
||||
if out.Packages == nil {
|
||||
out.Packages = map[string][]string{}
|
||||
}
|
||||
out.Packages[dir] = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,6 +120,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
return json.RawMessage(`{"path":"module.json","content":"{\"module\":\"snake\"}"}`), nil
|
||||
case "go.sum":
|
||||
return json.RawMessage(`{"path":"go.sum","content":"x"}`), nil
|
||||
case "go.mod": // a mesh package required at the zero version, which no go.sum line names (design 47 H-A-157)
|
||||
return json.RawMessage(`{"path":"go.mod","content":"module snake\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n)\n"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"path":"x","absent":true}`), nil
|
||||
}
|
||||
@@ -129,12 +131,15 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
|
||||
if len(got.Manifests) != 1 || len(files) != 4 { // the one directory asked: its manifest, two lock files and the go.mod, never the others
|
||||
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
|
||||
}
|
||||
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
|
||||
t.Fatalf("read = %+v", got)
|
||||
}
|
||||
if p := got.Packages[""]; len(p) != 1 || p[0] != "git.novox.be/novox/mesh-sdk/go" {
|
||||
t.Fatalf("mesh packages %v, want mesh-sdk, which the go.mod requires", got.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRuns records the run events handed to it and answers err.
|
||||
|
||||
+5
-3
@@ -58,7 +58,8 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name
|
||||
}
|
||||
|
||||
// TestASourceRevisionMovingMeshSDKNamesItAndItsImporters (design 47 H §743; A-476a; ADR 0593 §2 to §4): the indexer whole against
|
||||
// the bus. Two Go modules of the catalogue, importer, whose go.sum requires mesh-sdk's Go module, and plain, which requires
|
||||
// the bus. Two Go modules of the catalogue, importer, whose go.mod requires mesh-sdk's Go module at the zero version (no
|
||||
// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires
|
||||
// nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit
|
||||
// changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository,
|
||||
// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package.
|
||||
@@ -67,7 +68,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
|
||||
catalogMerge = "3333333333333333333333333333333333333333"
|
||||
goSumContent = "git.novox.be/novox/mesh-sdk/go v0.1.14 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.14/go.mod h1:y=\n"
|
||||
goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n"
|
||||
sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
@@ -119,7 +120,8 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777"))
|
||||
forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
|
||||
files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"),
|
||||
"modules/importer/go.sum": goSumContent, "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}})
|
||||
"modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n",
|
||||
"modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}})
|
||||
regs := registerAvahi(t, ctx, js, "main")
|
||||
for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}},
|
||||
{V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}},
|
||||
|
||||
@@ -135,14 +135,32 @@ func NamedTree(manifest string, paths map[string]string) (string, error) {
|
||||
return NamedTreePrefix + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under
|
||||
// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
|
||||
// takes the version line of (ADR 0593 §2).
|
||||
func MeshPackagesOf(goSum []byte) []string {
|
||||
// MeshRequiresOf are the mesh Go modules a go.mod requires (the module path of each `require` line under
|
||||
// trunk.ForgeModulePrefix, alone or in a block, at any version, the zero version and `// indirect` included), sorted and
|
||||
// each once; nil for none. They are a module's mesh packages from the take-in, with no build needed (design 47 H-A-157):
|
||||
// a go.sum names none required at the zero version (ADR 0573 §2), so it is never read for them.
|
||||
func MeshRequiresOf(goMod []byte) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(string(goSum), "\n") {
|
||||
inBlock := false
|
||||
for _, line := range strings.Split(string(goMod), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if i := strings.Index(line, "//"); i >= 0 {
|
||||
line = strings.TrimSpace(line[:i])
|
||||
}
|
||||
switch {
|
||||
case line == "require (":
|
||||
inBlock = true
|
||||
continue
|
||||
case inBlock && line == ")":
|
||||
inBlock = false
|
||||
continue
|
||||
case strings.HasPrefix(line, "require "):
|
||||
line = strings.TrimSpace(strings.TrimPrefix(line, "require "))
|
||||
case !inBlock:
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
|
||||
if len(f) != 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := trunk.PackageRepository(f[0]); ok {
|
||||
|
||||
@@ -31,6 +31,20 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) {
|
||||
if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun {
|
||||
t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got)
|
||||
}
|
||||
// Its mesh packages are every mesh package its go.mod requires, at any version, the zero version included (design 47
|
||||
// H-A-157); a replace line or a module outside the forge is none.
|
||||
for _, c := range []struct {
|
||||
goMod string
|
||||
want []string
|
||||
}{
|
||||
{string(goMod), []string{buildrun, sdk}},
|
||||
{"module x\n\nrequire " + sdk + " v0.0.0-00010101000000-000000000000 // indirect\nrequire github.com/nats-io/nats.go v1.37.0\n", []string{sdk}},
|
||||
{"module x\n\nreplace " + sdk + " => ../sdk\n", nil},
|
||||
} {
|
||||
if got := MeshRequiresOf([]byte(c.goMod)); strings.Join(got, ",") != strings.Join(c.want, ",") {
|
||||
t.Errorf("mesh packages of %q: %v, want %v", c.goMod, got, c.want)
|
||||
}
|
||||
}
|
||||
published := func(digest string) map[string]meshcatalog.PackagesByModuleValue {
|
||||
return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1",
|
||||
Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}}
|
||||
|
||||
@@ -108,7 +108,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
|
||||
if next.Manifest == "" {
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
} else {
|
||||
packages := ix.goSumPackages(ctx, f, dir, cur, has)
|
||||
packages := ix.goPackages(ctx, f, dir, cur, has)
|
||||
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
|
||||
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
|
||||
if in.Kept != nil && has && cur.Inputs2 != nil {
|
||||
@@ -365,79 +365,72 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
|
||||
return in
|
||||
}
|
||||
|
||||
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
|
||||
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
|
||||
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
others := ix.otherPackages(ctx, f, dir, cur, has)
|
||||
// goPackages are the version line's packages of the merge's directory dir (ADR 0593 §2; design 47 H-A-157): the mesh Go
|
||||
// modules of other repositories its go.mod requires, at any version, and those of its own repository its go.mod replaces
|
||||
// by a directory (siblingPackages, H-A-136's sibling rule). A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
|
||||
return others
|
||||
return nil
|
||||
}
|
||||
ix.requirementsOf(ctx, f, dir, cur, has)
|
||||
goMod, read := ix.goModOf(ctx, f, dir)
|
||||
others := fingerprint.OtherPackages(f.Packages[dir], f.Repository) // carried: what the go.mod requires, read with the tree
|
||||
switch {
|
||||
case len(others) > 0:
|
||||
case read:
|
||||
others = fingerprint.OtherPackages(fingerprint.MeshRequiresOf(goMod), f.Repository)
|
||||
case has && cur.Inputs2 != nil:
|
||||
others = fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
var siblings []string
|
||||
if read {
|
||||
siblings = fingerprint.ReplacedSiblings(goMod, f.Repository)
|
||||
} else if has && cur.Inputs2 != nil {
|
||||
for _, p := range cur.Inputs2.Packages {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) {
|
||||
siblings = append(siblings, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
siblings := ix.siblingPackages(ctx, f, dir, cur, has)
|
||||
if len(siblings) == 0 {
|
||||
return others
|
||||
}
|
||||
return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(others), siblings...))))
|
||||
}
|
||||
|
||||
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
|
||||
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
held := func() []string {
|
||||
if !has || cur.Inputs2 == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, p := range cur.Inputs2.Packages {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// goModOf is the go.mod of the merge's directory dir, read from the forge (GoMod), and whether it was read; where it
|
||||
// cannot be, the held module's packages stand, said.
|
||||
func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bool) {
|
||||
if ix.GoMod == nil {
|
||||
return held()
|
||||
return nil, false
|
||||
}
|
||||
raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the directory replaces it held: %v", f.Repository, f.Merge, dir, err)
|
||||
return held()
|
||||
ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
|
||||
return nil, false
|
||||
}
|
||||
return fingerprint.ReplacedSiblings(raw, f.Repository)
|
||||
return raw, true
|
||||
}
|
||||
|
||||
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
|
||||
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
|
||||
// from the forge (GoSum).
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
if carried := f.Packages[dir]; len(carried) > 0 {
|
||||
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
|
||||
}
|
||||
// requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state
|
||||
// (keepRequirements): read from the forge (GoSum) when no mesh package was carried and its hash moved, or this process
|
||||
// has not kept it yet. It gives no package: those are the go.mod's (H-A-157).
|
||||
func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) {
|
||||
hash := f.Locks[dir]["go.sum"]
|
||||
if hash == "" {
|
||||
return nil
|
||||
if len(f.Packages[dir]) > 0 || hash == "" || ix.GoSum == nil {
|
||||
return
|
||||
}
|
||||
if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) {
|
||||
for _, h := range cur.Inputs2.Locks {
|
||||
if h == hash {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if ix.GoSum == nil {
|
||||
if has && cur.Inputs2 != nil {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
|
||||
if has && cur.Inputs2 != nil {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
return nil
|
||||
ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err)
|
||||
return
|
||||
}
|
||||
ix.keepRequirements(ctx, f.Repository, dir, raw)
|
||||
return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository)
|
||||
}
|
||||
|
||||
@@ -402,6 +402,14 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means
|
||||
|
||||
func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6
|
||||
ix, mods, _ := manifestsIndexer(t) // no build record of any module
|
||||
// Its go.mod requires mesh-sdk at the zero version, which its go.sum does not name (ADR 0573 §2): the go.mod's
|
||||
// require lines are its mesh packages, with no build (design 47 H-A-157).
|
||||
ix.GoMod = func(context.Context, string, string, string) ([]byte, error) {
|
||||
return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil
|
||||
}
|
||||
ix.GoSum = func(context.Context, string, string, string) ([]byte, error) {
|
||||
return []byte("github.com/nats-io/nats.go v1.37.0 h1:x=\n"), nil
|
||||
}
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
@@ -412,6 +420,9 @@ func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *
|
||||
if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != meshcatalogclient.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" {
|
||||
t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok)
|
||||
}
|
||||
if !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) {
|
||||
t.Errorf("mesh packages %v, want mesh-sdk, which its go.mod requires at the zero version", m.Inputs2.Packages)
|
||||
}
|
||||
if m.Fp != "" || m.None != "" {
|
||||
t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m)
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func (r Requirement) Versions() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it
|
||||
// RequiredOf is, per mesh Go module a go.sum names (a line under trunk.ForgeModulePrefix whose path names a repository), the highest version it
|
||||
// names, its module line or its go.mod line alike.
|
||||
func RequiredOf(goSum []byte) map[string]string {
|
||||
out := map[string]string{}
|
||||
|
||||
@@ -9,7 +9,7 @@ type RepoRead struct {
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
// Packages are the mesh packages each directory's go.mod requires (fingerprint.MeshRequiresOf, design 47 H-A-157).
|
||||
Packages map[string][]string
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user