Design 47 H1: the indexer publishes a repository's packages in the merge's tree (H-A-232, H-A-233, H-A-234)

A Go module mesh-catalog removed stayed in the packages state, so every
publish of mesh-catalog asked the forge for its directory, the seat git
answered not-found, and the one-directory fallback (issue 1585) ran only
on an answer without the directory: nothing of the repository was
published and its packages' digests froze unsaid.

filesVia takes the seat's not-found as a failure of the tree, so the
fallback runs; publish drops the kept key of a package whose directory
the trunk's merge no longer holds, said in the journal; a listing that
fails for another reason raises the finding packages-not-published
(scope merge, id the repository), cleared at the next publish that lists.

Builds: design 47 H H-A-232, H-A-233, H-A-234; issue 1585
This commit is contained in:
2026-11-21 02:54:58 +01:00
parent 2cb6971194
commit ced27928d9
4 changed files with 182 additions and 8 deletions
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"slices"
@@ -115,10 +116,15 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string)
}
// filesVia asks the forge for every blob under each directory at a commit (the seat git's trees with files, ADR 0593 §2),
// by directory, then path relative to it. An answer that lists no directory asked is an error naming it.
// by directory, then path relative to it. An answer that lists no directory asked is an error naming it. The seat answers
// a directory the commit does not hold with the kind not-found naming it (the contract's files field; design 47 H H-A-232):
// that is a failure of the tree too (index.ErrNotInTree), so the indexer reads each directory alone and leaves the gone one out.
func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
a, err := treesOf(call, repository, git.TreesInput{Commit: sha, Files: dirs})
if errors.Is(err, index.ErrNotFound) {
return nil, fmt.Errorf("%w: %w", err, index.ErrNotInTree)
}
if err != nil {
return nil, err
}
@@ -75,6 +75,20 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
}
// H-A-232 (2): the seat's not-found naming a directory the commit lacks is a failure of the tree too, so the
// one-directory fallback runs; a failed listing of another kind is not.
gone := func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"error":"the directory modules/mesh-build-go/buildstep is not in novox/mesh-catalog at x","error-kind":"not-found"}`), nil
}
if _, err := filesVia(gone)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/mesh-build-go/buildstep"}); !errors.Is(err, index.ErrNotInTree) {
t.Errorf("a directory the seat says not-found: %v, want ErrNotInTree", err)
}
failed := func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"error":"the forge did not answer","error-kind":"failed"}`), nil
}
if _, err := filesVia(failed)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/a"}); err == nil || errors.Is(err, index.ErrNotInTree) {
t.Errorf("a listing that failed: %v, want an error that is not ErrNotInTree", err)
}
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
@@ -14,6 +14,8 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
@@ -138,8 +140,14 @@ func reached(held map[string]meshcatalog.ModulesByModuleValue) (modules, imports
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
// directory by directory). A module the listing has no go.mod for is said and left out.
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, error) {
out, _, err := ix.packagesIn(ctx, repository, commit, modules, imports)
return out, err
}
// packagesIn is packagesAt, with the modules whose directory the tree at the commit does not hold (gone), by module path.
func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, []string, error) {
if ix.Files == nil || commit == "" {
return nil, nil
return nil, nil, nil
}
dirOf := map[string]string{} // module path to its directory in the repository
for _, m := range modules {
@@ -182,7 +190,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
_, dir, _ := trunk.PackageRepository(imp)
for d := dir; ; d = parent(d) {
if err := list([]string{d}); err != nil {
return nil, err
return nil, nil, err
}
if _, ok := listings[d]["go.mod"]; ok {
m := strings.TrimSuffix(trunk.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
@@ -202,26 +210,28 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
dirs := slices.Sorted(maps.Values(dirOf))
if err := list(dirs); err != nil {
if !errors.Is(err, ErrNotInTree) || len(dirs) < 2 {
return nil, err
return nil, nil, err
}
whole := err
for _, d := range dirs {
if err := list([]string{d}); err != nil {
if !errors.Is(err, ErrNotInTree) {
return nil, err
return nil, nil, err
}
ix.logf("%s at %s: the directory %q is not in the tree, so the module there is not read: %v", repository, commit, d, err)
absent[d] = true
}
}
if len(absent) == len(dirs) {
return nil, whole // the tree itself fails
return nil, nil, whole // the tree itself fails
}
}
out := map[string]meshcatalog.PackagesByModuleValue{}
var gone []string
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
dir := dirOf[m]
if absent[dir] {
gone = append(gone, m)
continue
}
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
@@ -236,7 +246,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
out[m] = meshcatalog.PackagesByModuleValue{Module: m, Repository: trunk.RepositoryName(repository), Dir: d, Commit: commit,
Requires: requires, Packages: pkgs}
}
return out, nil
return out, gone, nil
}
// parent is the directory above d, "" for the repository's root.
@@ -287,10 +297,28 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
modules[repo] = append(modules[repo], m)
}
}
got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo])
got, gone, err := ix.packagesIn(ctx, repo, commit, modules[repo], imports[repo])
if err != nil {
ix.notPublished(ctx, repo, commit, err)
return err
}
ix.published(ctx, repo, commit)
// A repository's packages are those in the merge's tree (design 47 H H-A-232): the kept key of a package whose directory
// the trunk's merge no longer holds is dropped, said, so the forge is never asked for it again.
for _, m := range gone {
if _, kept := have[m]; !kept {
continue
}
key, err := PackagesKey(m)
if err != nil {
ix.logf("%s: %v", m, err)
continue
}
if err := write(ctx, ix.Packages, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
return fmt.Errorf("packages.%s: %w", key, err)
}
ix.logf("package %s of %s: its directory is gone at %s: its kept key dropped", m, repo, commit)
}
for _, m := range slices.Sorted(maps.Keys(got)) {
next := got[m]
key, err := PackagesKey(m)
@@ -316,6 +344,36 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
return nil
}
// PackagesNotPublishedKind is the finding of a repository whose packages a take-in's publish could not list, for a reason
// that is not a gone directory (design 47 H H-A-233, H-A-234): scope merge, id the repository, with the forge's words;
// raised at once and cleared at the repository's next publish that lists.
const PackagesNotPublishedKind = "packages-not-published"
// notPublished raises the finding packages-not-published of repository.
func (ix *Indexer) notPublished(ctx context.Context, repository, commit string, cause error) {
if ix.Findings == nil {
return
}
f := ix.observed(mcc.ScopeMerge, repository, PackagesNotPublishedKind,
fmt.Sprintf("the packages of %s are not published at %.12s", repository, commit),
"A repository's packages are not published",
fmt.Sprintf("The take-in of %s at %s could not list its packages (%v), so their digests stay at their last published commit and a change to one does not move its importers.", repository, commit, cause),
"indexer."+PackagesNotPublishedKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("%s: the finding %s is not written: %v", repository, PackagesNotPublishedKind, err)
}
}
// published clears the finding packages-not-published of repository: its publish listed.
func (ix *Indexer) published(ctx context.Context, repository, commit string) {
if ix.Findings == nil {
return
}
if err := ix.clearFinding(ctx, mcc.ScopeMerge, repository, PackagesNotPublishedKind, "its packages are listed at "+commit); err != nil {
ix.logf("%s: the finding %s is not cleared: %v", repository, PackagesNotPublishedKind, err)
}
}
// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
// A repository with no taken merge is left for its next merge.
@@ -0,0 +1,96 @@
package index
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// Design 47 H H-A-232 (1), H-A-233, H-A-234: a repository's packages are those in the merge's tree. The kept key of a
// package whose directory the trunk's merge no longer holds is dropped, said, and the repository's other packages are
// published all the same (issue 1585's one-directory fallback); a listing that fails for another reason raises the
// finding packages-not-published (scope merge, id the repository), cleared at the next publish that lists.
func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) {
const (
live = "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
gone = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
liveDir = "modules/disk-load/diskreadings"
)
ctx := context.Background()
old, merge := strings.Repeat("d", 40), strings.Repeat("e", 40)
held := map[string]meshcatalog.ModulesByModuleValue{
"disk-load": {Repository: catalogueRepo, Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{live}}},
}
var down error // the forge's listing fails for a reason that is not a gone directory
files := func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
if down != nil {
return nil, down
}
out := map[string]map[string]string{}
for _, d := range dirs {
if d != liveDir {
// as filesVia says the seat's not-found naming the directory
return nil, fmt.Errorf("the directory %s is not in %s at %s: %w", d, repository, sha, ErrNotInTree)
}
out[d] = map[string]string{"go.mod": "m1", "readings.go": "r-" + sha[:1]}
}
return out, nil
}
var said []string
findings := newFindings()
ix := &Indexer{Module: testSelf, Packages: newMem(), Files: files, Findings: findings,
Logf: func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }}
for _, m := range []struct{ module, dir string }{{live, liveDir}, {gone, "modules/mesh-build-go/buildstep"}} {
key, err := PackagesKey(m.module)
if err != nil {
t.Fatal(err)
}
raw, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: m.module, Repository: catalogueRepo, Dir: m.dir, Commit: old, Requires: "req", At: ptr(time.Unix(0, 0).UTC()),
Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: m.module, Digest: "x"}}})
if err != nil {
t.Fatal(err)
}
ix.Packages.(*memStore).put(key, raw)
}
notPublished := func() bool {
for _, f := range findings.Held() {
if f.Kind == PackagesNotPublishedKind && f.Scope == mcc.ScopeMerge && f.ID == keyID(catalogueRepo) {
return true
}
}
return false
}
down = errors.New("seat:git.trees (failed): the forge did not answer")
if err := ix.publish(ctx, catalogueRepo, merge, held, false); err == nil {
t.Fatal("a listing that failed published")
}
if !notPublished() {
t.Fatalf("a listing that failed raised no finding %s: %v", PackagesNotPublishedKind, findings.Held())
}
down = nil
if err := ix.publish(ctx, catalogueRepo, merge, held, false); err != nil {
t.Fatal(err)
}
have := ix.publishedAll(ctx)
if have[live].Commit != merge {
t.Errorf("packages of %s at %q, want the merge %s: a gone sibling kept it from being published", live, have[live].Commit, merge)
}
if _, kept := have[gone]; kept {
t.Errorf("the kept key of %s, whose directory the merge does not hold, stays", gone)
}
want := "package " + gone + " of " + catalogueRepo + ": its directory is gone at " + merge + ": its kept key dropped"
if !strings.Contains(strings.Join(said, "\n"), want) {
t.Errorf("the journal does not say %q:\n%s", want, strings.Join(said, "\n"))
}
if notPublished() {
t.Error("the finding packages-not-published stays after a publish that listed")
}
}