diff --git a/contracts/go/seats/meshcatalog/contract.go b/contracts/go/seats/meshcatalog/contract.go index db822342..dc2f44d9 100644 --- a/contracts/go/seats/meshcatalog/contract.go +++ b/contracts/go/seats/meshcatalog/contract.go @@ -680,13 +680,22 @@ type ActingRule struct { Step string `json:"step"` } +// ArtifactPackage is the package's name read from the revision's own files at the artifact's path (the module of its go.mod, the name of its package.json, the manifest's image name), written at take-in (design 47 G section 676). +type ArtifactPackage struct { + // go, npm or image + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` +} + // Artifact is one artifact of a module in the catalog graph entry. type Artifact struct { // the artifact's kind Kind string `json:"kind,omitempty"` // the artifact's language as its manifest says it (go, typescript, ...); a step's build seat is read from it (design 47 G section 323) - Language string `json:"language"` - Ref ArtifactRef `json:"ref"` + Language string `json:"language"` + // the package's name read from the revision's own files at the artifact's path (the module of its go.mod, the name of its package.json, the manifest's image name), written at take-in (design 47 G section 676) + Package *ArtifactPackage `json:"package,omitempty"` + Ref ArtifactRef `json:"ref"` } // ArtifactRef is an artifact reference, :, such as mesh-grants:program (design 47 *Dependencies between artifacts*). diff --git a/contracts/seats/mesh-catalog/seat.json b/contracts/seats/mesh-catalog/seat.json index 978e31dd..8b986d9b 100644 --- a/contracts/seats/mesh-catalog/seat.json +++ b/contracts/seats/mesh-catalog/seat.json @@ -2711,6 +2711,23 @@ "kind": { "type": "string", "description": "the artifact's kind" + }, + "package": { + "type": "object", + "description": "the package's name read from the revision's own files at the artifact's path (the module of its go.mod, the name of its package.json, the manifest's image name), written at take-in (design 47 G section 676)", + "properties": { + "ecosystem": { + "type": "string", + "description": "go, npm or image" + }, + "name": { + "type": "string" + } + }, + "required": [ + "ecosystem", + "name" + ] } }, "required": [ diff --git a/contracts/ts/seats/meshcatalog/index.d.ts b/contracts/ts/seats/meshcatalog/index.d.ts index 53832de7..9c74f608 100644 --- a/contracts/ts/seats/meshcatalog/index.d.ts +++ b/contracts/ts/seats/meshcatalog/index.d.ts @@ -191,12 +191,21 @@ export interface ActingRule { "step": string; } +/** the package's name read from the revision's own files at the artifact's path (the module of its go.mod, the name of its package.json, the manifest's image name), written at take-in (design 47 G section 676) */ +export interface ArtifactPackage { + /** go, npm or image */ + "ecosystem": string; + "name": string; +} + /** one artifact of a module in the catalog graph entry */ export interface Artifact { /** the artifact's kind */ "kind"?: string; /** the artifact's language as its manifest says it (go, typescript, ...); a step's build seat is read from it (design 47 G section 323) */ "language": string; + /** the package's name read from the revision's own files at the artifact's path (the module of its go.mod, the name of its package.json, the manifest's image name), written at take-in (design 47 G section 676) */ + "package"?: ArtifactPackage; "ref": ArtifactRef; } diff --git a/modules/mesh-delivery-indexer/internal/index/artifacts.go b/modules/mesh-delivery-indexer/internal/index/artifacts.go new file mode 100644 index 00000000..1e6c483e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/artifacts.go @@ -0,0 +1,95 @@ +package index + +import ( + "context" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/packagename" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Each entry carries its artifacts' package names (design 47 G §676, §491, §478, §504, §412): at take-in the indexer reads +// each artifact's package name from the revision's own files at the artifact's path, the module of its go.mod for a Go +// package, the name of its package.json for an npm package, the manifest's name of an image artifact, and writes it into +// the entry's artifact as package, {ecosystem, name}. The manifest validator judges §491 from these fields. + +// Package ecosystems of an artifact's package (§676). +const ( + EcosystemGo = "go" + EcosystemNpm = "npm" +) + +// ArtifactRefOf is the artifact reference of a module's artifact, :. +func ArtifactRefOf(module, artifact string) string { return module + ":" + artifact } + +// withArtifacts gives e its manifest's artifacts, by reference, with the package name of each package artifact (an image's name is an open design question: none is written). A +// name the forge does not answer is said and left out of the artifact, for the next entry to read again (the cache keeps only +// what was read); a manifest that does not decode leaves the entry as it is. +func (ix *Indexer) withArtifacts(ctx context.Context, e meshcatalog.CatalogEntry, m catalogue.Manifest) meshcatalog.CatalogEntry { + if m.Build == nil { + return e + } + for _, a := range m.Build.Artifacts { + art := meshcatalog.Artifact{Ref: ArtifactRefOf(e.Module, a.Name), Language: a.Language, Kind: a.Kind} + if pkg := ix.packageOf(ctx, e, a); pkg != nil { + art.Package = pkg + } + e.Artifacts = append(e.Artifacts, art) + } + slices.SortFunc(e.Artifacts, func(a, b meshcatalog.Artifact) int { return strings.Compare(a.Ref, b.Ref) }) + return e +} + +// packageOf is the package name of artifact a of the entry e: read through the forge at the entry's commit, kept per +// repository, commit, path and artifact so a later entry of the same commit reads nothing; nil for an artifact that names +// none (a program, an image, an archive) or whose name cannot be read now. +func (ix *Indexer) packageOf(ctx context.Context, e meshcatalog.CatalogEntry, a catalogue.Artifact) *meshcatalog.ArtifactPackage { + switch a.Kind { + case catalogue.ArtifactPackage: + default: + return nil + } + if ix.ReadFile == nil || e.Commit == "" { + return nil + } + ecosystem := EcosystemNpm + if a.Language == "go" { + ecosystem = EcosystemGo + } + key := e.Repository + "\x00" + e.Commit + "\x00" + e.Path + "\x00" + a.Name + ix.namesMu.Lock() + name, ok := ix.packageNames[key] + ix.namesMu.Unlock() + if !ok { + var err error + if name, err = packagename.Of(ctx, packagename.ReadFile(ix.ReadFile), e.Repository, e.Commit, e.Path, a); err != nil { + ix.logf("catalog graph: the package name of %s at %.12s is not read, so its entry carries none until the next entry: %v", ArtifactRefOf(e.Module, a.Name), e.Commit, err) + return nil + } + ix.namesMu.Lock() + if ix.packageNames == nil { + ix.packageNames = map[string]string{} + } + ix.packageNames[key] = name + ix.namesMu.Unlock() + } + return &meshcatalog.ArtifactPackage{Ecosystem: ecosystem, Name: name} +} + +// packageEntries is the entry of each module whose manifest decodes, with its artifacts' package names as the take-in reads +// them, for the judge of §491 to read the names from. +func (ix *Indexer) packageEntries(ctx context.Context, held map[string]meshcatalog.ModulesByModuleValue, raws map[string][]byte) []meshcatalog.CatalogEntry { + var out []meshcatalog.CatalogEntry + for _, name := range slices.Sorted(maps.Keys(raws)) { + m, err := catalogue.Decode(raws[name]) + if err != nil { + continue + } + cur := held[name] + out = append(out, ix.withArtifacts(ctx, meshcatalog.CatalogEntry{Module: name, Repository: cur.Repository, Path: cur.Path, Commit: cur.Commit}, m)) + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 1989d64e..fe7444a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -159,17 +159,19 @@ type Indexer struct { saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. - OwnBuild func(ctx context.Context) (string, error) - underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - judgeMu sync.Mutex // one judge pass at a time (verdicts.go) - ownMu sync.Mutex // guards ownFP - keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) - catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) - catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on - ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) - ownFixed bool // FixOwnBuild asked already: it asks once - ownSince time.Time // when the build was asked and not known, with ownWhy - ownWhy string + OwnBuild func(ctx context.Context) (string, error) + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + judgeMu sync.Mutex // one judge pass at a time (verdicts.go) + ownMu sync.Mutex // guards ownFP + keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) + namesMu sync.Mutex // packageNames + packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) + catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) + catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on + ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) + ownFixed bool // FixOwnBuild asked already: it asks once + ownSince time.Time // when the build was asked and not known, with ownWhy + ownWhy string // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go index fe9e5ae7..b2b1abf2 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go @@ -1,6 +1,8 @@ package index import ( + "context" + "reflect" "testing" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" @@ -29,3 +31,40 @@ func TestAnEntryCarriesItsManifestWholeAndTheDigestOfItsBytes(t *testing.T) { t.Errorf("an entry of before G read back holds %q, want %s (§609)", back.Manifest, digest) } } + +// Design 47 G §676: at take-in the entry's artifacts carry the package name read from the revision's own files at the +// artifact's path: the module of a go.mod, the name of a package.json. +func TestAnEntryCarriesItsArtifactsPackageNames(t *testing.T) { + ix := &Indexer{ReadFile: func(_ context.Context, repository, sha, path string) ([]byte, error) { + files := map[string]string{ + "mods/m/go/go.mod": "module git.example/m/go\n\ngo 1.26\n", + "mods/m/js/package.json": `{"name":"@example/m"}`, + } + if repository != "r" || sha != "c1" { + t.Errorf("read %s %s %s, want the entry's repository and commit", repository, sha, path) + } + if f, ok := files[path]; ok { + return []byte(f), nil + } + return nil, nil + }} + m := catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{ + {Name: "gopkg", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}, + {Name: "jspkg", Kind: catalogue.ArtifactPackage, Language: "typescript", Paths: []string{"js"}}, + {Name: "img", Kind: catalogue.ArtifactImage}, + {Name: "prog", Kind: "bundle", Language: "go"}, + }}} + e := ix.withArtifacts(context.Background(), meshcatalog.CatalogEntry{Module: "m", Repository: "r", Path: "mods/m", Commit: "c1"}, m) + got := map[string]string{} + for _, a := range e.Artifacts { + name := "none" + if a.Package != nil { + name = a.Package.Ecosystem + " " + a.Package.Name + } + got[a.Ref] = name + } + want := map[string]string{"m:gopkg": "go git.example/m/go", "m:jspkg": "npm @example/m", "m:img": "none", "m:prog": "none"} + if !reflect.DeepEqual(got, want) { + t.Errorf("artifacts' packages %v, want %v (§676)", got, want) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 3d5eaa8b..7599555b 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -189,11 +189,11 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. entries := make(map[string]meshcatalog.CatalogEntry, len(held)) for name, cur := range held { - entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]) + entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]), edges[name].manifest) } for name, e := range edges { if _, ok := held[name]; !ok { - entries[name] = CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]) + entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]), e.manifest) } } return ix.keepEntries(ctx, entries) @@ -235,7 +235,7 @@ func (ix *Indexer) KeepView(ctx context.Context) error { m, raw = read, bytes } } - entries[name] = CatalogEntryOf(name, cur, m, raw, nodes[name]) + entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, m, raw, nodes[name]), m) } return ix.keepEntries(ctx, entries) } diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 22bc5764..987ee838 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -174,6 +174,18 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove for _, p := range unnamed { ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p) } + // A package's name belongs to one module (design 47 G §491, §676): the trunk's manifests only, since a candidate's + // revision is validated by the validator built from its own heads (§16). + if over == nil { + if refused == nil { + refused = map[string][]string{} + } + for module, p := range judge.PackageNameProblems(ix.packageEntries(ctx, held, raws)) { + if _, waiting := waits[module]; !waiting { + refused[module] = append(refused[module], p...) + } + } + } out.Refused = refused // A manifest with a field this judge does not know waits for a judge that may know it (clause 3 of the record on a // release's Approve; ADR 0578 §1): it is refused only when this program is the newest build of the indexer the trunk diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index 5db980e7..87353e68 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -553,3 +553,48 @@ type unreadable struct{ Store } func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) { return nil, 0, false, errors.New("the bus did not answer") } + +// Design 47 G §491, §676: two modules whose entries carry one Go module path are both refused by the trunk's judge pass, +// each naming both; a module with its own name is accepted. +func TestAPackageNameBelongsToOneModule(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1", + Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} + ix.ReadFile = func(_ context.Context, _, _, path string) ([]byte, error) { + if strings.HasSuffix(path, "go.mod") { + name := "git.example/own/" + strings.Split(path, "/")[0] + if strings.HasPrefix(path, "one/") || strings.HasPrefix(path, "two/") { + name = "git.example/shared" + } + return []byte("module " + name + "\n"), nil + } + return nil, nil + } + hold := func(module string) string { + raw := `{"module":"` + module + `","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}` + digest, err := ix.keepManifest(ctx, module, raw) + if err != nil { + t.Fatal(err) + } + value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest, + Repository: "novox/r", Path: module, Commit: "c"}) + if err != nil { + t.Fatal(err) + } + ix.Modules.(*memStore).put(module, value) + return digest + } + digests := map[string]bool{hold("one"): true, hold("two"): true, hold("three"): true} + j, err := ix.JudgeAll(ctx, digests) + if err != nil { + t.Fatal(err) + } + if len(j.Refused) != 2 || j.Refused["one"] == nil || j.Refused["two"] == nil || j.Refused["three"] != nil { + t.Fatalf("refused %v, want one and two", j.Refused) + } + for _, m := range []string{"one", "two"} { + if p := strings.Join(j.Refused[m], " "); !strings.Contains(p, "one and two") || !strings.Contains(p, "git.example/shared") { + t.Errorf("%s's problems %q do not name both modules and the package", m, p) + } + } +} diff --git a/modules/mesh-delivery-indexer/judge/packagenames.go b/modules/mesh-delivery-indexer/judge/packagenames.go new file mode 100644 index 00000000..2803fb21 --- /dev/null +++ b/modules/mesh-delivery-indexer/judge/packagenames.go @@ -0,0 +1,43 @@ +package judge + +import ( + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// PackageNameProblems refuses a revision in which two modules' entries carry the same package name in one ecosystem, naming +// both (design 47 G §491, §676): a name read as a module must name one. It judges the entries' package fields, which the +// indexer wrote at take-in from the revision's own files, and answers each owning module's problems, sorted. +func PackageNameProblems(entries []meshcatalog.CatalogEntry) map[string][]string { + owners := map[string][]string{} // ecosystem and name -> modules + for _, e := range entries { + for _, a := range e.Artifacts { + if a.Package == nil { + continue + } + key := a.Package.Ecosystem + " " + a.Package.Name + if !slices.Contains(owners[key], e.Module) { + owners[key] = append(owners[key], e.Module) + } + } + } + out := map[string][]string{} + for key, modules := range owners { + if len(modules) < 2 { + continue + } + slices.Sort(modules) + ecosystem, name, _ := strings.Cut(key, " ") + problem := fmt.Sprintf("the %s package name %s is carried by the artifacts of %s: a package's name belongs to one module (design 47 G section 491)", ecosystem, name, strings.Join(modules, " and ")) + for _, m := range modules { + out[m] = append(out[m], problem) + } + } + for m := range out { + slices.Sort(out[m]) + } + return out +} diff --git a/modules/mesh-delivery-indexer/judge/packagenames_test.go b/modules/mesh-delivery-indexer/judge/packagenames_test.go new file mode 100644 index 00000000..bea3e51d --- /dev/null +++ b/modules/mesh-delivery-indexer/judge/packagenames_test.go @@ -0,0 +1,41 @@ +package judge + +import ( + "maps" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §491: two modules whose entries carry one Go module path are both refused, naming both; one name in two +// ecosystems, or one module's two artifacts of one name, is no problem. +func TestPackageNameProblems(t *testing.T) { + pkg := func(module, ecosystem, name string) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: module, Artifacts: []meshcatalog.Artifact{{Ref: module + ":p", Package: &meshcatalog.ArtifactPackage{Ecosystem: ecosystem, Name: name}}}} + } + cases := []struct { + name string + entries []meshcatalog.CatalogEntry + want []string + }{ + {"two modules, one Go name", []meshcatalog.CatalogEntry{pkg("a", "go", "x/y"), pkg("b", "go", "x/y"), pkg("c", "go", "x/z")}, []string{"a", "b"}}, + {"one name, two ecosystems", []meshcatalog.CatalogEntry{pkg("a", "go", "x"), pkg("b", "npm", "x")}, nil}, + {"one module twice", []meshcatalog.CatalogEntry{{Module: "a", Artifacts: []meshcatalog.Artifact{ + {Ref: "a:p", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x"}}, {Ref: "a:q", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x"}}}}}, nil}, + } + for _, c := range cases { + got := PackageNameProblems(c.entries) + modules := slices.Sorted(maps.Keys(got)) + if !reflect.DeepEqual(modules, c.want) && (len(modules) > 0 || len(c.want) > 0) { + t.Errorf("%s: refused %v, want %v", c.name, modules, c.want) + } + for m, problems := range got { + if len(problems) != 1 || !strings.Contains(problems[0], strings.Join(c.want, " and ")) { + t.Errorf("%s: %s's problems %q do not name %v", c.name, m, problems, c.want) + } + } + } +}