diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index c905eac1..c4311fa4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -1,5 +1,5 @@ -// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a -// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The +// Command mesh-delivery-indexer-tools serves the indexer's tools (novox/hq issue 1945): the runtime launches it for a +// call. A hand act writes one value of the indexer's takeins state and waits for the indexer's answer there. The // indexer is the other program: it serves no request subject (ADR 0437 §4); indexer_judge asks the same way, and // indexer_verdicts reads the indexer's verdicts state. The read tools of the indexer's states are // mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates). @@ -47,48 +47,6 @@ func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCoun } return map[string]any{"sections": c.Counts, "reads": c.Reads}, nil }}, - {Name: index.TakeInTool, - Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " + - "build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " + - "writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " + - "repository the forge could not read (a 404 before the forge's account could read it). A module the indexer holds " + - "whose record stands behind the head the gate's source names (a merge whose merged message never reached the " + - "indexer) is read again at that head and written when it changed. Answers stored, from which " + - "repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " + - "The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " + - "indexer_module (mesh-delivery-view) reads the result. With repository instead of module it takes in the head the gate's " + - "source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " + - "tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " + - "is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " + - "that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " + - "catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " + - "every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " + - "after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.", - Input: map[string]any{"type": "object", "properties": map[string]any{ - "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, - "repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"}, - "why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - var key string - var value []byte - var err error - switch { - case str(args, "repository") == "" && str(args, "module") == "": - key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now()) - case str(args, "repository") != "" && str(args, "module") == "": - key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now()) - default: - key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now()) - } - if err != nil { - return nil, err - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - }}, {Name: index.SeatDefinitionsAgainTool, Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " + "holds the module at, and keep them in the seat-definitions state as a take-in would (seat-definition-changed said where a digest moved; the trunk's seat-definition-unparsed and " + diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index fb8c3900..a9171c21 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -2,7 +2,7 @@ // bridge's messages on the stream FORGE and keeps the candidate groups (candidates.): the pull requests that share // a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed // directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its -// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches. +// hand-act tools (the program mesh-delivery-indexer-tools) ask through its state takeins, which it watches. // // Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that // same build again; this paragraph was added only to give the indexer a new build (issue 1933). @@ -384,8 +384,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) } } done := step("reading the build log") - // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each - // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). + // The log's entries at the start are read before any fingerprint is computed (below). var started atomic.Bool var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published if err := calls.ReadLog(builds.BuilderModule, builds.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error { @@ -394,9 +393,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil } if started.Load() { - if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil { - logger.Printf("the registered module %s is not indexed: %v", le.Key, err) - } // A trunk build's generated files out of step with their seat definitions (DLV-I20). ix.ApplyOutOfStepBuild(ctx, le.Value) // A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are. @@ -497,21 +493,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("computing the new-form fingerprints at start: %v", err) } done() - done = step("indexing the registered modules") - // A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names - // it and the bucket holds none (registered.go); the entries that follow are heard above. - if err := ix.IndexRegisteredAll(ctx); err != nil { - logger.Printf("indexing the registered modules at start: %v", err) - } - done() - // A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched, - // as every other input. A module the forge could not read is also read again by itself on the next source revision and - // merged message (index.RetryUnread). + // A person's hand act writes the takeins state (issue 1945): watched, as every other input. // A state not on the bus yet at this start is watched once it appears (issue 1790). - process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) { + process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) } if tw, err := settle(ctx, kv, eachTakeIn); err != nil { - process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again") + process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "no hand act of the indexer is answered until the indexer starts again") } else { go func() { errs <- follow(ctx, tw, eachTakeIn) }() } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 609e1b42..14625a36 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -13,8 +13,6 @@ import ( // takerIn is the part of the indexer a takeins value asks through. type takerIn interface { - TakeIn(ctx context.Context, module string) index.TakenIn - TakeInRepository(ctx context.Context, repository string) index.TakenIn SeatDefinitionsAgain(ctx context.Context, module string) index.TakenIn FingerprintsAgain(ctx context.Context, module string) index.TakenIn RecomputeByHand(ctx context.Context, by, why string) index.TakenIn @@ -28,8 +26,8 @@ type takerIn interface { HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn } -// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the -// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value +// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and +// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value // answered already (this indexer's own write) or deleted is left. func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { if e.Operation() != jetstream.KeyValuePut { @@ -74,13 +72,10 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream a.Module = r.Module } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) - } else if a.Repository != "" { - asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) - a.Module = a.Repository } else if a.SeatDefinitions { asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module) } else { - r = ix.TakeIn(ctx, a.Module) + r = index.TakenIn{Module: a.Module, Answer: "no hand act is named: the ask is left unanswered by any act"} } a.Result, a.Answered = &r, time.Now().UTC() logf("%s: a %s asked by %s (%s): %s", a.Module, asked, a.By, a.Why, r.Answer) diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 95acab90..d7c3286b 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -2,8 +2,6 @@ package index import ( "fmt" - "maps" - "slices" "strings" "sync" "time" @@ -15,7 +13,7 @@ import ( // Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads // of past builds: the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no -// fingerprint, and the places a module registered from its own repository was built from (RegistrationsOf). It is filled +// fingerprint. It is filled // from the log's entries at the indexer's start (every one the log keeps, 30 days) and then each new one as the builder // appends it; no entry is ever taken out, since only age takes one from the log. // @@ -64,6 +62,11 @@ func newer(a, b buildrecord.BuildRecord) bool { return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID) } +// succeeded is whether the record is an outcome that did not fail. +func succeeded(r buildrecord.BuildRecord) bool { + return r.State == "" && r.Failed == "" +} + // PathOf is the directory of a module of the repository as its newest build record of that repository names it, its // newest successful trunk build's first ("." for the repository's root), and false when no record of the repository // names the module (issue 1614). @@ -76,7 +79,7 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { if r.Module != module || !trunk.SameRepository(r.Repository, repository) { continue } - t := registered(r) && trunkLike[r.Ref] + t := succeeded(r) && trunkLike[r.Ref] if best == nil || (t && !onTrunk) || (t == onTrunk && newer(r, *best)) { c := r best, onTrunk = &c, t @@ -87,101 +90,3 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { } return pathOf(strings.Trim(best.Path, "/")), true } - -// Registration is a module registered in the catalogue from its own repository, as the builder log.s newest end of a build -// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was -// built at. -type Registration struct { - Module, Repository, Path, Commit, Ref string -} - -// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a -// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were -// built from nox-mesh, and every record of them names that ref). -func registered(r buildrecord.BuildRecord) bool { - return r.State == "" && r.Failed == "" -} - -// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each -// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's). -// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the -// first that holds the module's manifest is the one indexed. -func (b *Builds) RegistrationsOf(module string) []Registration { - type place struct{ repository, path string } - byPlace := map[place][]buildrecord.BuildRecord{} - b.mu.Lock() - for _, r := range b.records { - if r.Module == module && registered(r) { - k := place{trunk.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))} - byPlace[k] = append(byPlace[k], r) - } - } - b.mu.Unlock() - // Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a - // trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel - // and link2pay were built from nox-mesh), the newer of two equal; then its newest record. - best := map[place]buildrecord.BuildRecord{} - for k, recs := range byPlace { - count, latest := map[string]int{}, map[string]buildrecord.BuildRecord{} - for _, r := range recs { - count[r.Ref]++ - if cur, ok := latest[r.Ref]; !ok || newer(r, cur) { - latest[r.Ref] = r - } - } - var pick string - first := true - for ref, rec := range latest { - switch { - case first: - case trunkLike[ref] != trunkLike[pick]: - if !trunkLike[ref] { - continue - } - case count[ref] != count[pick]: - if count[ref] < count[pick] { - continue - } - case !newer(rec, latest[pick]): - continue - } - pick, first = ref, false - } - best[k] = latest[pick] - } - recs := slices.Collect(maps.Values(best)) - slices.SortFunc(recs, func(x, y buildrecord.BuildRecord) int { - switch { - case trunkLike[x.Ref] != trunkLike[y.Ref]: - if trunkLike[x.Ref] { - return -1 - } - return 1 - case newer(x, y): - return -1 - case newer(y, x): - return 1 - } - return strings.Compare(x.Repository+x.Path, y.Repository+y.Path) - }) - out := make([]Registration, 0, len(recs)) - for _, r := range recs { - out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref}) - } - return out -} - -// Registered is the registrations of every module a record names, by module name. -func (b *Builds) Registered() []Registration { - b.mu.Lock() - names := map[string]bool{} - for _, r := range b.records { - names[r.Module] = true - } - b.mu.Unlock() - var out []Registration - for _, name := range slices.Sorted(maps.Keys(names)) { - out = append(out, b.RegistrationsOf(name)...) - } - return out -} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index fa387d83..c010a78e 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -148,12 +148,7 @@ type Indexer struct { floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread moving map[string]whole // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu freshMu sync.Mutex recompute sync.Mutex // one Recompute at a time @@ -234,7 +229,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f whole) (Applied, error) { return applied, err } } - ix.kickRetry(ctx) // a registered module the forge could not read is read again on a take-in, in the background (issue 1945) lateMerge = w.late != "" } // The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's. diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 893036c6..fc5afbd7 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -467,13 +467,9 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { repository string // modules.a's repository at the end, "" for none deletedKeyA bool // Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge - // reads at the head of `to`; takeIn is a repository taken in after the merges (indexer_take_in), whose answer must - // hold answer and be stored or not. + // reads at the head of `to`. heads bool trunk []string - takeIn string - answer string - stored bool bGoneAtEnd bool }{ {name: "named while its trunk holds it: left", merges: []whole{held, named}, repository: from}, @@ -484,12 +480,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { merges: []whole{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true}, {name: "no module.json left, no other trunk holds it: deleted", merges: []whole{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true}, - {name: "taken in by repository, a head with no module.json: handed over from the other trunk", - merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true}, - {name: "taken in by repository, a head with modules: answered by name", - merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from}, - {name: "taken in by repository, no head named: answered so", - merges: []whole{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from}, } { t.Run(tc.name, func(t *testing.T) { mods := &deletesSeen{memStore: newMem()} @@ -502,9 +492,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { if repository == to && sha == toHead { return tc.trunk, nil } - if repository == from && tc.takeIn == from { - return nil, nil - } return nil, nil } ix.Read = func(_ context.Context, repository, sha, dir string) (RepoRead, error) { @@ -519,12 +506,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { t.Fatal(err) } } - if tc.takeIn != "" { - got := ix.TakeInRepository(context.Background(), tc.takeIn) - if got.Stored != tc.stored || !strings.Contains(got.Answer, tc.answer) { - t.Errorf("take-in answered %+v, want stored %v and %q", got, tc.stored, tc.answer) - } - } m, ok := moduleOf(t, mods.memStore, "a") if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository { t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs) diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go deleted file mode 100644 index 7f52fad2..00000000 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ /dev/null @@ -1,461 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "fmt" - "maps" - "path" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" -) - -// A module registered in the catalogue from its own repository (the operator registers it with the repository and the -// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge -// followed the registration. The controller's outcome records name each registered module's repository, directory and -// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the -// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged -// message as for every other repository. - -// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each -// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them. -type RepoRead struct { - Root string - Trees map[string]string - Manifests map[string]string - Locks map[string]map[string]string - // Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2). - Packages map[string][]string -} - -// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds -// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The -// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or -// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could -// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision -// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945). -func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { - _, err := ix.indexRegistered(ctx, r, false) - return err -} - -// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not. -type taken struct { - stored, held bool - repository, commit, dir string - why string -} - -func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) { - return ix.indexAt(ctx, r, force, false) -} - -// indexAt is indexRegistered; reread also reads a module the modules state holds (a person's take-in of a held module -// whose record stands behind the head the gate's source names, issue 1126). -func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bool) (taken, error) { - if ix.Modules == nil || ix.Read == nil { - return taken{why: "the indexer reads no forge or writes no modules"}, nil - } - key, err := meshcatalogclient.ModulesKey(r.Module) - if err != nil { - ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err) - return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil - } - if _, _, found, err := ix.Modules.Get(ctx, key); err != nil { - return taken{}, fmt.Errorf("modules.%s: %w", key, err) - } else if found && !reread { - ix.markRead(r.Module) - return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil - } - repository := trunk.RepositoryName(r.Repository) - commit := r.Commit - dir := strings.Trim(r.Path, "/") - if dir == meshcatalogclient.RootPath { - dir = "" - } - // The head the gate's source names for the repository, when it names one, else the registration's commit. A module a - // merge removed is absent at the head and is said and left. - ix.mu.Lock() - for repo, head := range ix.source.heads { - if trunk.SameRepository(repo, r.Repository) && head != "" { - commit = head - } - } - ix.mu.Unlock() - t := taken{repository: repository, commit: commit, dir: dir} - if repository == "" || commit == "" { - ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module) - t.why = "the controller's record names no repository or commit" - return t, nil - } - // One try for each repository, commit and directory: a repository with no manifest where it was registered is said once, - // not at every record of the controller's builds. A person's take-in (force) reads again. - tried := repository + "@" + commit + ":" + dir - ix.mu.Lock() - if ix.tried[tried] && !force { - ix.mu.Unlock() - t.why = "tried already at this commit and directory" - return t, nil - } - if ix.tried == nil { - ix.tried = map[string]bool{} - } - ix.tried[tried] = true - ix.mu.Unlock() - read, err := ix.Read(ctx, repository, commit, dir) - if err != nil { - ix.mu.Lock() - delete(ix.tried, tried) // a forge that could not say is asked again - if ix.unread == nil { - ix.unread = map[string]bool{} - } - ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread) - ix.mu.Unlock() - return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err) - } - ix.markRead(r.Module) - raw, ok := read.Manifests[dir] - if !ok || read.Trees[dir] == "" { - ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir) - t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir) - return t, nil - } - if name := manifestModule(raw); name != r.Module { - ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name) - t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name) - return t, nil - } - f := whole{Repository: repository, Merge: commit, Root: read.Root, - Trees: map[string]string{dir: read.Trees[dir]}, - Manifests: map[string]string{dir: raw}} - if l := read.Locks[dir]; len(l) > 0 { - f.Locks = map[string]map[string]string{dir: l} - } - if p := read.Packages[dir]; len(p) > 0 { - f.Packages = map[string][]string{dir: p} - } - if _, err := ix.applyManifests(ctx, f, false); err != nil { - return t, err - } - if _, _, found, err := ix.Modules.Get(ctx, key); err != nil { - return t, fmt.Errorf("modules.%s: %w", key, err) - } else if !found { - t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)" - return t, nil - } - t.stored = true - ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit) - return t, nil -} - -// markRead takes a module out of the unread ones: the forge answered for it. -func (ix *Indexer) markRead(module string) { - ix.mu.Lock() - delete(ix.unread, module) - ix.mu.Unlock() -} - -// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while -// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the -// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand -// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next. -func (ix *Indexer) RetryUnread(ctx context.Context) { - ix.mu.Lock() - names := slices.Sorted(maps.Keys(ix.unread)) - ix.mu.Unlock() - for _, module := range names { - if err := ix.IndexRegisteredModule(ctx, module); err != nil { - ix.logf("the registered module %s is still not indexed: %v", module, err) - } - } -} - -// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without -// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run -// once more when it ends; never two at once. -func (ix *Indexer) kickRetry(ctx context.Context) { - ix.mu.Lock() - none := len(ix.unread) == 0 - ix.mu.Unlock() - if none { - return - } - ix.retryAgain.Store(true) - if !ix.retrying.CompareAndSwap(false, true) { - return - } - ix.retryWG.Add(1) - go func() { - defer ix.retryWG.Done() - for ix.retryAgain.Swap(false) { - ix.RetryUnread(ctx) - } - ix.retrying.Store(false) - // A kick between the last Swap and the Store found it running and returned: run for it. - if ix.retryAgain.Load() { - ix.kickRetry(ctx) - } - }() -} - -// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from -// where, and otherwise why not. -type TakenIn struct { - Module string `json:"module"` - Stored bool `json:"stored"` - Held bool `json:"held,omitempty"` - Repository string `json:"repository,omitempty"` - Commit string `json:"commit,omitempty"` - Path string `json:"path,omitempty"` - Answer string `json:"answer"` - // Handed and Dropped are the modules a take-in of a repository (TakeInRepository) moved to another repository's trunk - // and deleted. - Handed []string `json:"handed,omitempty"` - Dropped []string `json:"dropped,omitempty"` - // Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read. - Unread []string `json:"unread,omitempty"` - // Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new. - Fingerprints []FingerprintMove `json:"fingerprints,omitempty"` - // Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON. - Registry json.RawMessage `json:"registry,omitempty"` -} - -// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool -// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the -// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why. -func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn { - out := TakenIn{Module: module} - if again, ok := ix.takeInHeld(ctx, module); ok { - return again - } - var regs []Registration - if ix.Builds != nil { - regs = ix.Builds.RegistrationsOf(module) - } - if len(regs) == 0 { - if r, why, ok := ix.registeredAtBranch(ctx, module); ok { - regs = []Registration{r} - } else if why != "" { - ix.logf("take-in of %s: %s", module, why) - out.Answer = why - return out - } - } - if len(regs) == 0 { - if key, err := meshcatalogclient.ModulesKey(module); err == nil && ix.Modules != nil { - if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found { - out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on" - return out - } - } - out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge" - return out - } - var whys []string - for _, r := range regs { - t, err := ix.indexRegistered(ctx, r, true) - out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir) - switch { - case err != nil: - whys = append(whys, err.Error()) - continue - case t.held: - out.Held, out.Answer = true, t.why - out.Repository, out.Commit, out.Path = "", "", "" - return out - case t.stored: - out.Stored = true - out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit) - return out - } - whys = append(whys, t.why) - } - out.Answer = "not taken in: " + strings.Join(whys, "; ") - return out -} - -// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one. -func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error { - if ix.Builds == nil { - return nil - } - var first error - for _, r := range ix.Builds.RegistrationsOf(module) { - if err := ix.IndexRegistered(ctx, r); err != nil && first == nil { - first = err - } - } - return first -} - -// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is -// said and the others go on; the first error is answered. -func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error { - if ix.Builds == nil { - return nil - } - var first error - for _, r := range ix.Builds.Registered() { - if err := ix.IndexRegistered(ctx, r); err != nil { - ix.logf("the registered module %s is not indexed: %v", r.Module, err) - if first == nil { - first = err - } - } - } - return first -} - -// takeInHeld reads again, at the head the gate's source names for its repository, a module the modules state holds whose -// record stands at another commit, and writes it as a merge would (only when it changed). It is the recovery for a merge -// whose merged message never reached the indexer (issue 1126: mesh-catalog#535's message was given up by the forge bridge -// after its reads timed out, and mesh-vault's record stayed at the merge before). It answers false where it does not -// apply: no such module held, or no head named, or the record at that head already. -func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) { - out := TakenIn{Module: module} - key, err := meshcatalogclient.ModulesKey(module) - if err != nil || ix.Modules == nil { - return out, false - } - value, _, found, err := ix.Modules.Get(ctx, key) - if err != nil || !found { - return out, false - } - cur, err := meshcatalogclient.DecodeModule(value) - if err != nil || cur.Repository == "" || cur.Path == "" { - return out, false - } - head := "" - ix.mu.Lock() - for repo, h := range ix.source.heads { - if trunk.SameRepository(repo, cur.Repository) && h != "" { - head = h - } - } - ix.mu.Unlock() - if head == "" || head == cur.Commit { - return out, false - } - // Never backwards (order.go): a source head older than the record (a merge applied before the source moved) is never - // read and written back over it. A diverged or rewritten history, or a record whose commit the forge no longer has, is - // not older and is read; where the forge cannot say otherwise, nothing is read. One forge call. - old, why := true, "the forge cannot be asked" - if ix.Contains != nil { - var err error - if old, err = ix.older(ctx, cur.Repository, head, cur.Commit); err != nil { - old, why = true, err.Error() - } else if old { - why = fmt.Sprintf("%.12s is older than its record at %.12s", head, cur.Commit) - } - } - if old { - out.Held, out.Repository, out.Commit, out.Path = true, cur.Repository, cur.Commit, cur.Path - out.Answer = fmt.Sprintf("held; the source names another head, %.12s, and not one newer than its record (%s), so it is not read again", head, why) - return out, true - } - t, err := ix.indexAt(ctx, Registration{Module: module, Repository: cur.Repository, Path: cur.Path, Commit: head}, true, true) - out.Held, out.Repository, out.Commit, out.Path = true, t.repository, t.commit, pathOf(t.dir) - switch { - case err != nil: - out.Answer = "held, and not read again at the source's head: " + err.Error() - case !t.stored: - out.Answer = "held, and not read again at the source's head: " + t.why - default: - after, _, _, _ := ix.Modules.Get(ctx, key) - out.Stored = true - if string(after) == string(value) { - out.Answer = fmt.Sprintf("held, and read again from %s at %s: unchanged there, so its record stands", t.repository, t.commit) - } else { - out.Answer = fmt.Sprintf("held, read again from %s at %s and written to the modules state: it changed since %.12s", t.repository, t.commit, cur.Commit) - } - } - return out, true -} - -// TakeInRepository takes in the head the gate's source names for a repository whose merged message the indexer never -// heard (the tool indexer_take_in with repository, issue 1367). A head whose trunk holds no module.json is applied as -// the merged message of a merge that leaves none: every module of the repository goes, one another trunk holds handed over -// from there first. A head whose trunk holds modules is answered "by name": each is taken in with its own name. -func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) TakenIn { - out := TakenIn{Repository: repository} - ix.mu.Lock() - head := "" - for repo, h := range ix.source.heads { - if trunk.SameRepository(repo, repository) && h != "" { - head, out.Repository = h, repo - } - } - ix.mu.Unlock() - if head == "" || ix.Modules == nil || ix.Trunk == nil { - out.Answer = "the gate's source names no head of " + repository + ", or the indexer reads no forge or writes no modules" - return out - } - out.Commit = head - dirs, err := ix.Trunk(ctx, out.Repository, head) - if err != nil { - out.Answer = fmt.Sprintf("the modules of %s at %.12s cannot be read, so nothing is taken in: %v", repository, head, err) - return out - } - if len(dirs) > 0 { - out.Answer = fmt.Sprintf("%s at %.12s holds modules (%s): take each module in by name (module)", repository, head, strings.Join(dirs, ", ")) - return out - } - before, err := ix.heldModules(ctx) - if err != nil { - out.Answer = "the modules state cannot be read: " + err.Error() - return out - } - f := whole{Repository: out.Repository, Merge: head} - if _, err := ix.ApplyMessage(ctx, f); err != nil { - out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err) - return out - } - after, err := ix.heldModules(ctx) - if err != nil { - out.Answer = "the modules state cannot be read: " + err.Error() - return out - } - for _, key := range slices.Sorted(maps.Keys(before)) { - if !trunk.SameRepository(before[key].Repository, repository) { - continue - } - if now, ok := after[key]; ok { - out.Handed = append(out.Handed, key+" to "+now.Repository+" at "+now.Commit) - } else { - out.Dropped = append(out.Dropped, key) - } - } - out.Stored = true - out.Answer = fmt.Sprintf("%s at %.12s holds no module.json: taken in as a merge that leaves none; handed over %v, dropped %v", repository, head, out.Handed, out.Dropped) - return out -} - -// registeredAtBranch is the module's registration (the seat mesh-catalog's, ADR 0666) read at the head of its first watched -// branch, which is its trunk (design 47 G A-85): a module from outside the indexed repositories, watched on main, master or -// a branch its owner chose (nox-mesh), is taken in from there when the controller's build records name none. why says what -// stopped it; none and not ok is no registration. -func (ix *Indexer) registeredAtBranch(ctx context.Context, module string) (Registration, string, bool) { - if ix.Registry == nil || ix.Registry.Registrations == nil || ix.BranchHead == nil { - return Registration{}, "", false - } - reg, found, err := ix.Registry.Registration(ctx, module) - if err != nil { - return Registration{}, fmt.Sprintf("the registration of %s cannot be read: %v", module, err), false - } - if !found { - return Registration{}, "", false - } - if len(reg.Branches) == 0 || reg.Origin == "" { - return Registration{}, fmt.Sprintf("the registration of %s names no repository or watched branch", module), false - } - branch := reg.Branches[0] - head, err := ix.BranchHead(ctx, reg.Origin, branch) - if err != nil || head == "" { - return Registration{}, fmt.Sprintf("the head of %s's branch %s cannot be read: %v", reg.Origin, branch, err), false - } - ix.logf("take-in of %s: registered from %s, read at the head %s of its first watched branch %s", module, reg.Origin, head, branch) - return Registration{Module: module, Repository: reg.Origin, Path: path.Dir(reg.Path), Commit: head, Ref: branch}, "", true -} diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go deleted file mode 100644 index c7f035b8..00000000 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ /dev/null @@ -1,326 +0,0 @@ -package index - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here - const forge = "http://novox.internal:20000/jschoubben/snake.git" - snake := `{"module":"snake","version":"1.0"}` - for _, tc := range []struct { - name string - path string - read RepoRead - err error - ref string // the build record's ref - heads map[string]string // the gate source's heads - wantRead bool // the forge is read - held bool // modules.snake exists already - other bool // modules.other of the same repository exists - want bool // modules.snake is written - }{ - {wantRead: true, name: "a module at the root of its repository", path: "", want: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a root spelled as a dot", path: ".", want: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a module in a directory", path: "modules/snake", want: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}}, - {wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "a module the bucket holds is left", path: "", held: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "no manifest at the path", path: "modules/x", - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a manifest of another module", path: "", - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1.0"}`}}}, - {wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2}, - read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")}, - } { - t.Run(tc.name, func(t *testing.T) { - ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", "")) - if tc.held { - if err := mods.Create(context.Background(), "snake", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil { - t.Fatal(err) - } - } - if tc.other { - if err := mods.Create(context.Background(), "other", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil { - t.Fatal(err) - } - } - ix.Manifests = newMem() - var asked struct { - repo, sha, dir string - n int - } - ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) { - asked.repo, asked.sha, asked.dir = repo, sha, dir - asked.n++ - return tc.read, tc.err - } - if tc.heads != nil { - ix.source = sourceNow{rev: 1, heads: tc.heads} - } - err := ix.IndexRegisteredAll(context.Background()) - if (err != nil) != (tc.err != nil) { - t.Fatalf("error = %v", err) - } - wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's - if tc.heads != nil { - wantCommit = h2 // the source's head - } - if tc.wantRead && asked.sha != wantCommit { - t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit) - } - wantReads := 1 - if !tc.wantRead { - wantReads = 0 - } - if tc.wantRead && asked.dir != dirOf(tc.path) { - t.Errorf("the forge was asked for directory %q", asked.dir) - } - if asked.n != wantReads { - t.Errorf("forge reads = %d, want %d", asked.n, wantReads) - } - m, ok := moduleOf(t, mods, "snake") - if tc.want { - if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) { - t.Fatalf("modules.snake = %+v, found %v", m, ok) - } - if len(ix.Manifests.(*memStore).vals) != 1 { - t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals)) - } - if asked.repo != "jschoubben/snake" { - t.Errorf("the forge was asked for %q", asked.repo) - } - } else if !tc.held && ok { - t.Fatalf("modules.snake written: %+v", m) - } - if _, ok := moduleOf(t, mods, "other"); ok != tc.other { - t.Errorf("modules.other held = %v, want %v", ok, tc.other) - } - }) - } - - // Issue 1126: mesh-catalog#535 changed mesh-vault, but its merged message never reached the indexer, so the record - // stood at the merge before. A take-in of the held module reads it again at the source's head, only when the record is - // behind that head: a head older than the record (a merge applied before the source moved), or one whose order the forge - // cannot say, is never written back. - for _, tc := range []struct { - name string - contains func(context.Context, string, string, string) (bool, error) - moved bool - }{ - {name: "a held module behind the source's head is read again", moved: true, - contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == h2, nil }}, - {name: "a source head older than the record is not read", contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == h2 && in == merge, nil }}, - // Review of 77d738d: a diverged or rewritten history, or a record whose commit the forge no longer has, is not older. - {name: "a source head of a diverged history is read again", moved: true, - contains: func(context.Context, string, string, string) (bool, error) { return false, nil }}, - {name: "a record whose commit the forge no longer has is read again", moved: true, contains: func(context.Context, string, string, string) (bool, error) { - return false, fmt.Errorf("seat:git.contains (not-found): the forge holds no commit: %w", ErrNotFound) // the kind, no status in its words - }}, - {name: "a forge that cannot say the order reads nothing", contains: func(context.Context, string, string, string) (bool, error) { - return false, errors.New("timeout") - }}, - {name: "no forge to ask reads nothing"}, - } { - t.Run("take-in: "+tc.name, func(t *testing.T) { - ctx := context.Background() - ix, mods := modulesIndexer(t) - ix.Manifests = newMem() - if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "treeA"})); err != nil { - t.Fatal(err) - } - ix.source = sourceNow{rev: 2, heads: map[string]string{"novox/mesh-catalog": h2}} - ix.Contains = tc.contains - var asked []string - ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) { - asked = append(asked, sha+":"+dir) - return RepoRead{Root: "r1", Trees: map[string]string{"modules/a": "treeA2"}, Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}, nil - } - got := ix.TakeIn(ctx, "a") - m, ok := moduleOf(t, mods, "a") - if !tc.moved { - if got.Stored || !got.Held || len(asked) != 0 || !strings.Contains(got.Answer, "another head") || !ok || m.Commit != merge { - t.Errorf("take-in answered %+v, asked %v, modules.a at %.8s", got, asked, m.Commit) - } - return - } - if !got.Stored || !got.Held || got.Commit != h2 || got.Path != "modules/a" || !strings.Contains(got.Answer, "written") { - t.Errorf("take-in answered %+v", got) - } - if !ok || m.Commit != h2 || m.Inputs2 == nil || m.Inputs2.Tree != "treeA2" { - t.Errorf("modules.a = %+v (%v), want it at %.8s with tree treeA2 (asked %v)", m, ok, h2, asked) - } - n := len(asked) // at the head now: a take-in answers held and reads nothing - if again := ix.TakeIn(ctx, "a"); again.Stored || !again.Held || len(asked) != n { - t.Errorf("a take-in of a module at the source's head answered %+v and asked %v", again, asked) - } - }) - } -} - -func mustModule(t *testing.T, m meshcatalog.ModulesByModuleValue) []byte { - t.Helper() - b, err := meshcatalogclient.EncodeModule(withV(m)) - if err != nil { - t.Fatal(err) - } - return b -} - -func dirOf(p string) string { - if p == "." { - return "" - } - return p -} - -func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh - const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git" - ix, mods := modulesIndexer(t, - rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""), - rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""), - rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", "")) - ix.Manifests = newMem() - var asked []string - ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) { - asked = append(asked, repo+":"+dir) - if repo == "novox/Link2Pay" { - return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1.0"}`}}, nil - } - return RepoRead{Root: "r0"}, nil - } - if err := ix.IndexRegisteredAll(context.Background()); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "link2pay") - if !ok || m.Repository != "novox/Link2Pay" { - t.Fatalf("modules.link2pay = %+v, found %v", m, ok) - } - if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" { - t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked) - } -} - -func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit - // Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when - // the indexer started again. Each row's trigger must now take the module in once the forge reads it. - const snake = `{"module":"snake","version":"1.0"}` - ctx := context.Background() - for _, tc := range []struct{ failure, trigger string }{ - {"timeout", "the indexer starts again"}, - {"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"}, - {"Gitea API /repos/x/git/trees/y: 404 not found", "a take-in of another repository"}, - {"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"}, - {"timeout", "a person's take-in"}, - } { - t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) { - ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", "")) - ix.Manifests = newMem() - ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}} - var asked []string - readable := false - ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) { - asked = append(asked, sha) - if readable { - return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil - } - return RepoRead{}, errors.New(tc.failure) - } - if err := ix.IndexRegisteredAll(ctx); err == nil { - t.Errorf("no error") - } - if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 { - t.Errorf("written %v, asked %v", ok, asked) - } - if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 { - t.Errorf("not asked again (asked %v)", asked) - } - readable = true // the forge's account reads the repository now - var got TakenIn - switch tc.trigger { - case "the indexer starts again": - if err := ix.IndexRegisteredAll(ctx); err != nil { - t.Fatal(err) - } - case "a take-in of another repository": - if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil { - t.Fatal(err) - } - ix.retryWG.Wait() // read again in the background - case "a person's take-in": - got = ix.TakeIn(ctx, "snake") - if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" { - t.Errorf("take-in answered %+v", got) - } - } - if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 { - t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked) - } - // Taken in, it is read no more: a further event asks the forge nothing. - n := len(asked) - if err := applyS(ix, ctx, 3, srcT{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil { - t.Fatal(err) - } - ix.retryWG.Wait() - if len(asked) != n { - t.Errorf("the forge was asked again after the module was taken in: %v", asked) - } - if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n { - t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked) - } - if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") { - t.Errorf("a take-in of an unregistered module answered %+v", none) - } - }) - } -} - -func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place - const repo = "http://novox.internal:20000/novox/photos.git" - b := NewBuilds() - for _, r := range [][2]any{ - rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""), - rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""), - rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""), - } { - _ = b.Put(r[1].([]byte)) - } - got := b.RegistrationsOf("photos") - if len(got) != 1 || got[0].Ref != "nox-mesh" { - t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got) - } - k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "") - _, _ = k, b.Put(v) // a trunk's ref beats a branch of its own - if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" { - t.Fatalf("registrations = %+v", got) - } - // An entry of another kind in the builder's log (the old reference form's close, design 51), of this form or another, - // is no build: passed over, never refused. - for _, other := range []string{ - fmt.Sprintf(`{"v":%d,"event":"old-form-closed","module":"node-engine","at":"2026-10-04T09:00:00Z"}`, keys.Version), - `{"v":99,"event":"old-form-closed","at":"2026-10-04T09:00:00Z","nodes":["ace"]}`, - } { - if err := b.Put([]byte(other)); err != nil { - t.Errorf("an entry of another kind is refused: %v", err) - } - } - if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" { - t.Fatalf("after entries of another kind, registrations = %+v", got) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 419faf50..24ba847e 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -17,7 +17,7 @@ import ( // The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a // module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions, // indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the -// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject. +// indexer by an ask in takeins, as every hand act does, since the indexer's process serves no request subject. const ( CatalogRevisionTool = "indexer_catalog_revision" CatalogRevisionsTool = "indexer_catalog_revisions" diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index fb701ba5..d815fc46 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -7,33 +7,24 @@ import ( "regexp" "strings" "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" ) -// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in -// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool -// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's -// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools -// of the indexer's states stay in mesh-delivery-view. +// The indexer's hand acts (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): the tools bundle +// serves them through the tool runner; the indexer's process serves no request subject (ADR 0437 §4), so a tool writes +// an ask into the indexer's state takeins, which the indexer watches and answers there, as the builder answers +// builder_ask_again. No tool takes a head in by hand: the gate's gate_head_take is the one hand take of a head (design 47 +// H §742, H-A-105). const ( - TakeInTool = "indexer_take_in" - // TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state. + // TakeInsState is the indexer's state of hand-act asks: the bucket mesh-delivery-indexer_takeins. TakeInsState = "takeins" ) -// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the -// indexer read it, its answer. +// TakeInAsk is a value of the takeins state: which hand act a person asked, why, who and when, and, once the indexer read +// it, its answer. type TakeInAsk struct { V int `json:"v"` Module string `json:"module,omitempty"` - // Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module. - Repository string `json:"repository,omitempty"` - Why string `json:"why,omitempty"` - // CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool, - // indexer_take_in called with no module and no repository, as the node agent calls it after a restore. - CatchUp bool `json:"catch-up,omitempty"` + Why string `json:"why,omitempty"` // SeatDefinitions asks a contracts module's trunk seat definitions to be read again now (indexer_seat_definitions_again, // DLV-I19). SeatDefinitions bool `json:"seat-definitions,omitempty"` @@ -91,51 +82,12 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil { + if a.Module == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil { return a, errors.New("a takeins value naming no module, repository, pending wait or merge") } return a, nil } -// NewTakeIn is the value indexer_take_in writes for a module: its key and the value. -func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) { - module = strings.TrimSpace(module) - if module == "" { - return "", nil, errors.New("name the module to take in (module)") - } - key, err := meshcatalogclient.ModulesKey(module) - if err != nil { - return "", nil, err - } - data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) - return key, data, err -} - -// NewTakeInRepository is the value indexer_take_in writes for a repository: its key and the value. -func NewTakeInRepository(repository, why, by string, at time.Time) (string, []byte, error) { - repository = strings.TrimSpace(repository) - key, err := trunk.RepositoryKey(repository) - if err != nil { - return "", nil, err - } - data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) - return "repository." + key, data, err -} - -// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first. -const CatchUpKey = "catch-up" - -// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every -// repository the gate's source names (CatchUp). -func NewCatchUp(why, by string, at time.Time) (string, []byte, error) { - why = strings.TrimSpace(why) - if why == "" { - why = "the catch-up after a restore" - } - data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()}) - return CatchUpKey, data, err -} - // TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585). const TakeInMergeTool = "indexer_take_in_merge" diff --git a/modules/mesh-delivery-indexer/internal/index/takenin.go b/modules/mesh-delivery-indexer/internal/index/takenin.go new file mode 100644 index 00000000..e899a25f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takenin.go @@ -0,0 +1,33 @@ +package index + +import "encoding/json" + +// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each +// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them. +type RepoRead struct { + Root string + Trees map[string]string + Manifests map[string]string + Locks map[string]map[string]string + // Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2). + Packages map[string][]string +} + +// TakenIn is the answer of an ask of the state takeins: whether what was asked is done, of which module, from where, and +// otherwise why not. +type TakenIn struct { + Module string `json:"module"` + Stored bool `json:"stored"` + Held bool `json:"held,omitempty"` + Repository string `json:"repository,omitempty"` + Commit string `json:"commit,omitempty"` + Path string `json:"path,omitempty"` + Answer string `json:"answer"` + // Handed and Dropped are what a hand act on the catalog revisions moved and removed (revision_tools.go). + Handed []string `json:"handed,omitempty"` + Dropped []string `json:"dropped,omitempty"` + // Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new. + Fingerprints []FingerprintMove `json:"fingerprints,omitempty"` + // Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON. + Registry json.RawMessage `json:"registry,omitempty"` +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 497861cb..310c3228 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -140,11 +140,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, - // capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the - // catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves. + // capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1. Design 47 H + // §742 (H-A-105): no catch-up tool of the indexer's follows a restore, indexer_take_in having gone with the builder-log + // index; the gate's gate_head_take is the one hand take of a head. if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 || - m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) { - t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool) + m.Objects[0].Form != 1 || m.Objects[0].CatchUp != "" { + t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, no catch-up tool", m.Objects, index.CatalogRevisionsObjects) } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own // name and states with their revisions, and the states it reads, each as `.`. @@ -185,11 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). + // 1945: the hand acts are served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules // kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read // tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as). - want := []string{"indexer_take_in", "indexer_old_sections"} + want := []string{"indexer_old_sections"} for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) { want = append(want, v.Tool) // Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of. @@ -204,7 +205,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. @@ -224,7 +225,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("the indexer declares the state %s, which design 47 G retires", s.Name) } } - for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads"} { + for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads", "indexer_take_in"} { if slices.Contains(m.Tools, retired) { t.Errorf("the indexer serves %s, which design 47 G retires", retired) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 302f873b..b19bd00b 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -27,7 +27,6 @@ } ], "tools": [ - "indexer_take_in", "indexer_old_sections", "manifests", "modules", @@ -125,7 +124,7 @@ { "name": "mesh-delivery-indexer-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "objects": [ - { "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" } + { "name": "catalog-revisions", "max-mib": 512, "form": 1 } ], "emits": [ "take-in.pending",