H-A-48, H-A-105: the builder-log index and indexer_take_in go

TakeIn, TakeInRepository, the catch-up, IndexRegistered and RetryUnread go: a module is taken in
only from a source revision, and the gate's gate_head_take is the one hand take of a head. The
manifest serves no indexer_take_in and names no catch-up tool beside the catalog revisions.

Builds: design 47 H §742, §764 (the indexer); H-A-48, H-A-105 (a)

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:27 +01:00
parent 8d2d490637
commit 1ece9449fd
13 changed files with 72 additions and 1054 deletions
@@ -1,5 +1,5 @@
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
// Command mesh-delivery-indexer-tools serves the indexer's tools (novox/hq issue 1945): the runtime launches it for a
// call. A hand act writes one value of the indexer's takeins state and waits for the indexer's answer there. The
// indexer is the other program: it serves no request subject (ADR 0437 §4); indexer_judge asks the same way, and
// indexer_verdicts reads the indexer's verdicts state. The read tools of the indexer's states are
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
@@ -47,48 +47,6 @@ func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCoun
}
return map[string]any{"sections": c.Counts, "reads": c.Reads}, nil
}},
{Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
"repository the forge could not read (a 404 before the forge's account could read it). A module the indexer holds " +
"whose record stands behind the head the gate's source names (a merge whose merged message never reached the " +
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
"indexer_module (mesh-delivery-view) reads the result. With repository instead of module it takes in the head the gate's " +
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
"that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " +
"catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " +
"every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " +
"after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
var key string
var value []byte
var err error
switch {
case str(args, "repository") == "" && str(args, "module") == "":
key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now())
case str(args, "repository") != "" && str(args, "module") == "":
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
default:
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
}
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: index.SeatDefinitionsAgainTool,
Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " +
"holds the module at, and keep them in the seat-definitions state as a take-in would (seat-definition-changed said where a digest moved; the trunk's seat-definition-unparsed and " +
@@ -2,7 +2,7 @@
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
// hand-act tools (the program mesh-delivery-indexer-tools) ask through its state takeins, which it watches.
//
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
@@ -384,8 +384,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
}
done := step("reading the build log")
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
// The log's entries at the start are read before any fingerprint is computed (below).
var started atomic.Bool
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
if err := calls.ReadLog(builds.BuilderModule, builds.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
@@ -394,9 +393,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
}
if started.Load() {
if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil {
logger.Printf("the registered module %s is not indexed: %v", le.Key, err)
}
// A trunk build's generated files out of step with their seat definitions (DLV-I20).
ix.ApplyOutOfStepBuild(ctx, le.Value)
// A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are.
@@ -497,21 +493,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
done()
done = step("indexing the registered modules")
// A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names
// it and the bucket holds none (registered.go); the entries that follow are heard above.
if err := ix.IndexRegisteredAll(ctx); err != nil {
logger.Printf("indexing the registered modules at start: %v", err)
}
done()
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
// merged message (index.RetryUnread).
// A person's hand act writes the takeins state (issue 1945): watched, as every other input.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) }
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again")
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "no hand act of the indexer is answered until the indexer starts again")
} else {
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
}
@@ -13,8 +13,6 @@ import (
// takerIn is the part of the indexer a takeins value asks through.
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
TakeInRepository(ctx context.Context, repository string) index.TakenIn
SeatDefinitionsAgain(ctx context.Context, module string) index.TakenIn
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
RecomputeByHand(ctx context.Context, by, why string) index.TakenIn
@@ -28,8 +26,8 @@ type takerIn interface {
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and
// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
// answered already (this indexer's own write) or deleted is left.
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
@@ -74,13 +72,10 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
a.Module = r.Module
} else if a.Fingerprints {
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
} else if a.Repository != "" {
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
a.Module = a.Repository
} else if a.SeatDefinitions {
asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module)
} else {
r = ix.TakeIn(ctx, a.Module)
r = index.TakenIn{Module: a.Module, Answer: "no hand act is named: the ask is left unanswered by any act"}
}
a.Result, a.Answered = &r, time.Now().UTC()
logf("%s: a %s asked by %s (%s): %s", a.Module, asked, a.By, a.Why, r.Answer)
@@ -2,8 +2,6 @@ package index
import (
"fmt"
"maps"
"slices"
"strings"
"sync"
"time"
@@ -15,7 +13,7 @@ import (
// Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads
// of past builds: the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no
// fingerprint, and the places a module registered from its own repository was built from (RegistrationsOf). It is filled
// fingerprint. It is filled
// from the log's entries at the indexer's start (every one the log keeps, 30 days) and then each new one as the builder
// appends it; no entry is ever taken out, since only age takes one from the log.
//
@@ -64,6 +62,11 @@ func newer(a, b buildrecord.BuildRecord) bool {
return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID)
}
// succeeded is whether the record is an outcome that did not fail.
func succeeded(r buildrecord.BuildRecord) bool {
return r.State == "" && r.Failed == ""
}
// PathOf is the directory of a module of the repository as its newest build record of that repository names it, its
// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository
// names the module (issue 1614).
@@ -76,7 +79,7 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
if r.Module != module || !trunk.SameRepository(r.Repository, repository) {
continue
}
t := registered(r) && trunkLike[r.Ref]
t := succeeded(r) && trunkLike[r.Ref]
if best == nil || (t && !onTrunk) || (t == onTrunk && newer(r, *best)) {
c := r
best, onTrunk = &c, t
@@ -87,101 +90,3 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
}
return pathOf(strings.Trim(best.Path, "/")), true
}
// Registration is a module registered in the catalogue from its own repository, as the builder log.s newest end of a build
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
// built at.
type Registration struct {
Module, Repository, Path, Commit, Ref string
}
// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
// built from nox-mesh, and every record of them names that ref).
func registered(r buildrecord.BuildRecord) bool {
return r.State == "" && r.Failed == ""
}
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
// first that holds the module's manifest is the one indexed.
func (b *Builds) RegistrationsOf(module string) []Registration {
type place struct{ repository, path string }
byPlace := map[place][]buildrecord.BuildRecord{}
b.mu.Lock()
for _, r := range b.records {
if r.Module == module && registered(r) {
k := place{trunk.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
byPlace[k] = append(byPlace[k], r)
}
}
b.mu.Unlock()
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
best := map[place]buildrecord.BuildRecord{}
for k, recs := range byPlace {
count, latest := map[string]int{}, map[string]buildrecord.BuildRecord{}
for _, r := range recs {
count[r.Ref]++
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
latest[r.Ref] = r
}
}
var pick string
first := true
for ref, rec := range latest {
switch {
case first:
case trunkLike[ref] != trunkLike[pick]:
if !trunkLike[ref] {
continue
}
case count[ref] != count[pick]:
if count[ref] < count[pick] {
continue
}
case !newer(rec, latest[pick]):
continue
}
pick, first = ref, false
}
best[k] = latest[pick]
}
recs := slices.Collect(maps.Values(best))
slices.SortFunc(recs, func(x, y buildrecord.BuildRecord) int {
switch {
case trunkLike[x.Ref] != trunkLike[y.Ref]:
if trunkLike[x.Ref] {
return -1
}
return 1
case newer(x, y):
return -1
case newer(y, x):
return 1
}
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
})
out := make([]Registration, 0, len(recs))
for _, r := range recs {
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
}
return out
}
// Registered is the registrations of every module a record names, by module name.
func (b *Builds) Registered() []Registration {
b.mu.Lock()
names := map[string]bool{}
for _, r := range b.records {
names[r.Module] = true
}
b.mu.Unlock()
var out []Registration
for _, name := range slices.Sorted(maps.Keys(names)) {
out = append(out, b.RegistrationsOf(name)...)
}
return out
}
@@ -148,12 +148,7 @@ type Indexer struct {
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]whole // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
freshMu sync.Mutex
recompute sync.Mutex // one Recompute at a time
@@ -234,7 +229,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f whole) (Applied, error) {
return applied, err
}
}
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a take-in, in the background (issue 1945)
lateMerge = w.late != ""
}
// The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's.
@@ -467,13 +467,9 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
repository string // modules.a's repository at the end, "" for none
deletedKeyA bool
// Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge
// reads at the head of `to`; takeIn is a repository taken in after the merges (indexer_take_in), whose answer must
// hold answer and be stored or not.
// reads at the head of `to`.
heads bool
trunk []string
takeIn string
answer string
stored bool
bGoneAtEnd bool
}{
{name: "named while its trunk holds it: left", merges: []whole{held, named}, repository: from},
@@ -484,12 +480,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
merges: []whole{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
{name: "no module.json left, no other trunk holds it: deleted",
merges: []whole{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
{name: "taken in by repository, a head with no module.json: handed over from the other trunk",
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
{name: "taken in by repository, a head with modules: answered by name",
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
{name: "taken in by repository, no head named: answered so",
merges: []whole{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
} {
t.Run(tc.name, func(t *testing.T) {
mods := &deletesSeen{memStore: newMem()}
@@ -502,9 +492,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
if repository == to && sha == toHead {
return tc.trunk, nil
}
if repository == from && tc.takeIn == from {
return nil, nil
}
return nil, nil
}
ix.Read = func(_ context.Context, repository, sha, dir string) (RepoRead, error) {
@@ -519,12 +506,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
t.Fatal(err)
}
}
if tc.takeIn != "" {
got := ix.TakeInRepository(context.Background(), tc.takeIn)
if got.Stored != tc.stored || !strings.Contains(got.Answer, tc.answer) {
t.Errorf("take-in answered %+v, want stored %v and %q", got, tc.stored, tc.answer)
}
}
m, ok := moduleOf(t, mods.memStore, "a")
if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository {
t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs)
@@ -1,461 +0,0 @@
package index
import (
"context"
"encoding/json"
"fmt"
"maps"
"path"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
// followed the registration. The controller's outcome records name each registered module's repository, directory and
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
// message as for every other repository.
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
type RepoRead struct {
Root string
Trees map[string]string
Manifests map[string]string
Locks map[string]map[string]string
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
Packages map[string][]string
}
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
_, err := ix.indexRegistered(ctx, r, false)
return err
}
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
type taken struct {
stored, held bool
repository, commit, dir string
why string
}
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
return ix.indexAt(ctx, r, force, false)
}
// indexAt is indexRegistered; reread also reads a module the modules state holds (a person's take-in of a held module
// whose record stands behind the head the gate's source names, issue 1126).
func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bool) (taken, error) {
if ix.Modules == nil || ix.Read == nil {
return taken{why: "the indexer reads no forge or writes no modules"}, nil
}
key, err := meshcatalogclient.ModulesKey(r.Module)
if err != nil {
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
} else if found && !reread {
ix.markRead(r.Module)
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
}
repository := trunk.RepositoryName(r.Repository)
commit := r.Commit
dir := strings.Trim(r.Path, "/")
if dir == meshcatalogclient.RootPath {
dir = ""
}
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
// merge removed is absent at the head and is said and left.
ix.mu.Lock()
for repo, head := range ix.source.heads {
if trunk.SameRepository(repo, r.Repository) && head != "" {
commit = head
}
}
ix.mu.Unlock()
t := taken{repository: repository, commit: commit, dir: dir}
if repository == "" || commit == "" {
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
t.why = "the controller's record names no repository or commit"
return t, nil
}
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
// not at every record of the controller's builds. A person's take-in (force) reads again.
tried := repository + "@" + commit + ":" + dir
ix.mu.Lock()
if ix.tried[tried] && !force {
ix.mu.Unlock()
t.why = "tried already at this commit and directory"
return t, nil
}
if ix.tried == nil {
ix.tried = map[string]bool{}
}
ix.tried[tried] = true
ix.mu.Unlock()
read, err := ix.Read(ctx, repository, commit, dir)
if err != nil {
ix.mu.Lock()
delete(ix.tried, tried) // a forge that could not say is asked again
if ix.unread == nil {
ix.unread = map[string]bool{}
}
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
ix.mu.Unlock()
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
}
ix.markRead(r.Module)
raw, ok := read.Manifests[dir]
if !ok || read.Trees[dir] == "" {
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
return t, nil
}
if name := manifestModule(raw); name != r.Module {
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
return t, nil
}
f := whole{Repository: repository, Merge: commit, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]},
Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
}
if p := read.Packages[dir]; len(p) > 0 {
f.Packages = map[string][]string{dir: p}
}
if _, err := ix.applyManifests(ctx, f, false); err != nil {
return t, err
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return t, fmt.Errorf("modules.%s: %w", key, err)
} else if !found {
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
return t, nil
}
t.stored = true
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
return t, nil
}
// markRead takes a module out of the unread ones: the forge answered for it.
func (ix *Indexer) markRead(module string) {
ix.mu.Lock()
delete(ix.unread, module)
ix.mu.Unlock()
}
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
func (ix *Indexer) RetryUnread(ctx context.Context) {
ix.mu.Lock()
names := slices.Sorted(maps.Keys(ix.unread))
ix.mu.Unlock()
for _, module := range names {
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
ix.logf("the registered module %s is still not indexed: %v", module, err)
}
}
}
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
// once more when it ends; never two at once.
func (ix *Indexer) kickRetry(ctx context.Context) {
ix.mu.Lock()
none := len(ix.unread) == 0
ix.mu.Unlock()
if none {
return
}
ix.retryAgain.Store(true)
if !ix.retrying.CompareAndSwap(false, true) {
return
}
ix.retryWG.Add(1)
go func() {
defer ix.retryWG.Done()
for ix.retryAgain.Swap(false) {
ix.RetryUnread(ctx)
}
ix.retrying.Store(false)
// A kick between the last Swap and the Store found it running and returned: run for it.
if ix.retryAgain.Load() {
ix.kickRetry(ctx)
}
}()
}
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
// where, and otherwise why not.
type TakenIn struct {
Module string `json:"module"`
Stored bool `json:"stored"`
Held bool `json:"held,omitempty"`
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Path string `json:"path,omitempty"`
Answer string `json:"answer"`
// Handed and Dropped are the modules a take-in of a repository (TakeInRepository) moved to another repository's trunk
// and deleted.
Handed []string `json:"handed,omitempty"`
Dropped []string `json:"dropped,omitempty"`
// Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read.
Unread []string `json:"unread,omitempty"`
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
// Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON.
Registry json.RawMessage `json:"registry,omitempty"`
}
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
out := TakenIn{Module: module}
if again, ok := ix.takeInHeld(ctx, module); ok {
return again
}
var regs []Registration
if ix.Builds != nil {
regs = ix.Builds.RegistrationsOf(module)
}
if len(regs) == 0 {
if r, why, ok := ix.registeredAtBranch(ctx, module); ok {
regs = []Registration{r}
} else if why != "" {
ix.logf("take-in of %s: %s", module, why)
out.Answer = why
return out
}
}
if len(regs) == 0 {
if key, err := meshcatalogclient.ModulesKey(module); err == nil && ix.Modules != nil {
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
return out
}
}
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
return out
}
var whys []string
for _, r := range regs {
t, err := ix.indexRegistered(ctx, r, true)
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
switch {
case err != nil:
whys = append(whys, err.Error())
continue
case t.held:
out.Held, out.Answer = true, t.why
out.Repository, out.Commit, out.Path = "", "", ""
return out
case t.stored:
out.Stored = true
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
return out
}
whys = append(whys, t.why)
}
out.Answer = "not taken in: " + strings.Join(whys, "; ")
return out
}
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
if ix.Builds == nil {
return nil
}
var first error
for _, r := range ix.Builds.RegistrationsOf(module) {
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
first = err
}
}
return first
}
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
// said and the others go on; the first error is answered.
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
if ix.Builds == nil {
return nil
}
var first error
for _, r := range ix.Builds.Registered() {
if err := ix.IndexRegistered(ctx, r); err != nil {
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
if first == nil {
first = err
}
}
}
return first
}
// takeInHeld reads again, at the head the gate's source names for its repository, a module the modules state holds whose
// record stands at another commit, and writes it as a merge would (only when it changed). It is the recovery for a merge
// whose merged message never reached the indexer (issue 1126: mesh-catalog#535's message was given up by the forge bridge
// after its reads timed out, and mesh-vault's record stayed at the merge before). It answers false where it does not
// apply: no such module held, or no head named, or the record at that head already.
func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) {
out := TakenIn{Module: module}
key, err := meshcatalogclient.ModulesKey(module)
if err != nil || ix.Modules == nil {
return out, false
}
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil || !found {
return out, false
}
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Repository == "" || cur.Path == "" {
return out, false
}
head := ""
ix.mu.Lock()
for repo, h := range ix.source.heads {
if trunk.SameRepository(repo, cur.Repository) && h != "" {
head = h
}
}
ix.mu.Unlock()
if head == "" || head == cur.Commit {
return out, false
}
// Never backwards (order.go): a source head older than the record (a merge applied before the source moved) is never
// read and written back over it. A diverged or rewritten history, or a record whose commit the forge no longer has, is
// not older and is read; where the forge cannot say otherwise, nothing is read. One forge call.
old, why := true, "the forge cannot be asked"
if ix.Contains != nil {
var err error
if old, err = ix.older(ctx, cur.Repository, head, cur.Commit); err != nil {
old, why = true, err.Error()
} else if old {
why = fmt.Sprintf("%.12s is older than its record at %.12s", head, cur.Commit)
}
}
if old {
out.Held, out.Repository, out.Commit, out.Path = true, cur.Repository, cur.Commit, cur.Path
out.Answer = fmt.Sprintf("held; the source names another head, %.12s, and not one newer than its record (%s), so it is not read again", head, why)
return out, true
}
t, err := ix.indexAt(ctx, Registration{Module: module, Repository: cur.Repository, Path: cur.Path, Commit: head}, true, true)
out.Held, out.Repository, out.Commit, out.Path = true, t.repository, t.commit, pathOf(t.dir)
switch {
case err != nil:
out.Answer = "held, and not read again at the source's head: " + err.Error()
case !t.stored:
out.Answer = "held, and not read again at the source's head: " + t.why
default:
after, _, _, _ := ix.Modules.Get(ctx, key)
out.Stored = true
if string(after) == string(value) {
out.Answer = fmt.Sprintf("held, and read again from %s at %s: unchanged there, so its record stands", t.repository, t.commit)
} else {
out.Answer = fmt.Sprintf("held, read again from %s at %s and written to the modules state: it changed since %.12s", t.repository, t.commit, cur.Commit)
}
}
return out, true
}
// TakeInRepository takes in the head the gate's source names for a repository whose merged message the indexer never
// heard (the tool indexer_take_in with repository, issue 1367). A head whose trunk holds no module.json is applied as
// the merged message of a merge that leaves none: every module of the repository goes, one another trunk holds handed over
// from there first. A head whose trunk holds modules is answered "by name": each is taken in with its own name.
func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) TakenIn {
out := TakenIn{Repository: repository}
ix.mu.Lock()
head := ""
for repo, h := range ix.source.heads {
if trunk.SameRepository(repo, repository) && h != "" {
head, out.Repository = h, repo
}
}
ix.mu.Unlock()
if head == "" || ix.Modules == nil || ix.Trunk == nil {
out.Answer = "the gate's source names no head of " + repository + ", or the indexer reads no forge or writes no modules"
return out
}
out.Commit = head
dirs, err := ix.Trunk(ctx, out.Repository, head)
if err != nil {
out.Answer = fmt.Sprintf("the modules of %s at %.12s cannot be read, so nothing is taken in: %v", repository, head, err)
return out
}
if len(dirs) > 0 {
out.Answer = fmt.Sprintf("%s at %.12s holds modules (%s): take each module in by name (module)", repository, head, strings.Join(dirs, ", "))
return out
}
before, err := ix.heldModules(ctx)
if err != nil {
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
f := whole{Repository: out.Repository, Merge: head}
if _, err := ix.ApplyMessage(ctx, f); err != nil {
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
return out
}
after, err := ix.heldModules(ctx)
if err != nil {
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
for _, key := range slices.Sorted(maps.Keys(before)) {
if !trunk.SameRepository(before[key].Repository, repository) {
continue
}
if now, ok := after[key]; ok {
out.Handed = append(out.Handed, key+" to "+now.Repository+" at "+now.Commit)
} else {
out.Dropped = append(out.Dropped, key)
}
}
out.Stored = true
out.Answer = fmt.Sprintf("%s at %.12s holds no module.json: taken in as a merge that leaves none; handed over %v, dropped %v", repository, head, out.Handed, out.Dropped)
return out
}
// registeredAtBranch is the module's registration (the seat mesh-catalog's, ADR 0666) read at the head of its first watched
// branch, which is its trunk (design 47 G A-85): a module from outside the indexed repositories, watched on main, master or
// a branch its owner chose (nox-mesh), is taken in from there when the controller's build records name none. why says what
// stopped it; none and not ok is no registration.
func (ix *Indexer) registeredAtBranch(ctx context.Context, module string) (Registration, string, bool) {
if ix.Registry == nil || ix.Registry.Registrations == nil || ix.BranchHead == nil {
return Registration{}, "", false
}
reg, found, err := ix.Registry.Registration(ctx, module)
if err != nil {
return Registration{}, fmt.Sprintf("the registration of %s cannot be read: %v", module, err), false
}
if !found {
return Registration{}, "", false
}
if len(reg.Branches) == 0 || reg.Origin == "" {
return Registration{}, fmt.Sprintf("the registration of %s names no repository or watched branch", module), false
}
branch := reg.Branches[0]
head, err := ix.BranchHead(ctx, reg.Origin, branch)
if err != nil || head == "" {
return Registration{}, fmt.Sprintf("the head of %s's branch %s cannot be read: %v", reg.Origin, branch, err), false
}
ix.logf("take-in of %s: registered from %s, read at the head %s of its first watched branch %s", module, reg.Origin, head, branch)
return Registration{Module: module, Repository: reg.Origin, Path: path.Dir(reg.Path), Commit: head, Ref: branch}, "", true
}
@@ -1,326 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
)
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
const forge = "http://novox.internal:20000/jschoubben/snake.git"
snake := `{"module":"snake","version":"1.0"}`
for _, tc := range []struct {
name string
path string
read RepoRead
err error
ref string // the build record's ref
heads map[string]string // the gate source's heads
wantRead bool // the forge is read
held bool // modules.snake exists already
other bool // modules.other of the same repository exists
want bool // modules.snake is written
}{
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{name: "a module the bucket holds is left", path: "", held: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "no manifest at the path", path: "modules/x",
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a manifest of another module", path: "",
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1.0"}`}}},
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
} {
t.Run(tc.name, func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
if tc.held {
if err := mods.Create(context.Background(), "snake", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
t.Fatal(err)
}
}
if tc.other {
if err := mods.Create(context.Background(), "other", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
t.Fatal(err)
}
}
ix.Manifests = newMem()
var asked struct {
repo, sha, dir string
n int
}
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
asked.repo, asked.sha, asked.dir = repo, sha, dir
asked.n++
return tc.read, tc.err
}
if tc.heads != nil {
ix.source = sourceNow{rev: 1, heads: tc.heads}
}
err := ix.IndexRegisteredAll(context.Background())
if (err != nil) != (tc.err != nil) {
t.Fatalf("error = %v", err)
}
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
if tc.heads != nil {
wantCommit = h2 // the source's head
}
if tc.wantRead && asked.sha != wantCommit {
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
}
wantReads := 1
if !tc.wantRead {
wantReads = 0
}
if tc.wantRead && asked.dir != dirOf(tc.path) {
t.Errorf("the forge was asked for directory %q", asked.dir)
}
if asked.n != wantReads {
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
}
m, ok := moduleOf(t, mods, "snake")
if tc.want {
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
t.Fatalf("modules.snake = %+v, found %v", m, ok)
}
if len(ix.Manifests.(*memStore).vals) != 1 {
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
}
if asked.repo != "jschoubben/snake" {
t.Errorf("the forge was asked for %q", asked.repo)
}
} else if !tc.held && ok {
t.Fatalf("modules.snake written: %+v", m)
}
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
}
})
}
// Issue 1126: mesh-catalog#535 changed mesh-vault, but its merged message never reached the indexer, so the record
// stood at the merge before. A take-in of the held module reads it again at the source's head, only when the record is
// behind that head: a head older than the record (a merge applied before the source moved), or one whose order the forge
// cannot say, is never written back.
for _, tc := range []struct {
name string
contains func(context.Context, string, string, string) (bool, error)
moved bool
}{
{name: "a held module behind the source's head is read again", moved: true,
contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == h2, nil }},
{name: "a source head older than the record is not read", contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == h2 && in == merge, nil }},
// Review of 77d738d: a diverged or rewritten history, or a record whose commit the forge no longer has, is not older.
{name: "a source head of a diverged history is read again", moved: true,
contains: func(context.Context, string, string, string) (bool, error) { return false, nil }},
{name: "a record whose commit the forge no longer has is read again", moved: true, contains: func(context.Context, string, string, string) (bool, error) {
return false, fmt.Errorf("seat:git.contains (not-found): the forge holds no commit: %w", ErrNotFound) // the kind, no status in its words
}},
{name: "a forge that cannot say the order reads nothing", contains: func(context.Context, string, string, string) (bool, error) {
return false, errors.New("timeout")
}},
{name: "no forge to ask reads nothing"},
} {
t.Run("take-in: "+tc.name, func(t *testing.T) {
ctx := context.Background()
ix, mods := modulesIndexer(t)
ix.Manifests = newMem()
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "treeA"})); err != nil {
t.Fatal(err)
}
ix.source = sourceNow{rev: 2, heads: map[string]string{"novox/mesh-catalog": h2}}
ix.Contains = tc.contains
var asked []string
ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) {
asked = append(asked, sha+":"+dir)
return RepoRead{Root: "r1", Trees: map[string]string{"modules/a": "treeA2"}, Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}, nil
}
got := ix.TakeIn(ctx, "a")
m, ok := moduleOf(t, mods, "a")
if !tc.moved {
if got.Stored || !got.Held || len(asked) != 0 || !strings.Contains(got.Answer, "another head") || !ok || m.Commit != merge {
t.Errorf("take-in answered %+v, asked %v, modules.a at %.8s", got, asked, m.Commit)
}
return
}
if !got.Stored || !got.Held || got.Commit != h2 || got.Path != "modules/a" || !strings.Contains(got.Answer, "written") {
t.Errorf("take-in answered %+v", got)
}
if !ok || m.Commit != h2 || m.Inputs2 == nil || m.Inputs2.Tree != "treeA2" {
t.Errorf("modules.a = %+v (%v), want it at %.8s with tree treeA2 (asked %v)", m, ok, h2, asked)
}
n := len(asked) // at the head now: a take-in answers held and reads nothing
if again := ix.TakeIn(ctx, "a"); again.Stored || !again.Held || len(asked) != n {
t.Errorf("a take-in of a module at the source's head answered %+v and asked %v", again, asked)
}
})
}
}
func mustModule(t *testing.T, m meshcatalog.ModulesByModuleValue) []byte {
t.Helper()
b, err := meshcatalogclient.EncodeModule(withV(m))
if err != nil {
t.Fatal(err)
}
return b
}
func dirOf(p string) string {
if p == "." {
return ""
}
return p
}
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
ix, mods := modulesIndexer(t,
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
ix.Manifests = newMem()
var asked []string
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
asked = append(asked, repo+":"+dir)
if repo == "novox/Link2Pay" {
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1.0"}`}}, nil
}
return RepoRead{Root: "r0"}, nil
}
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "link2pay")
if !ok || m.Repository != "novox/Link2Pay" {
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
}
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
}
}
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
const snake = `{"module":"snake","version":"1.0"}`
ctx := context.Background()
for _, tc := range []struct{ failure, trigger string }{
{"timeout", "the indexer starts again"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "a take-in of another repository"},
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
{"timeout", "a person's take-in"},
} {
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
ix.Manifests = newMem()
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
var asked []string
readable := false
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
asked = append(asked, sha)
if readable {
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
}
return RepoRead{}, errors.New(tc.failure)
}
if err := ix.IndexRegisteredAll(ctx); err == nil {
t.Errorf("no error")
}
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
t.Errorf("written %v, asked %v", ok, asked)
}
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
t.Errorf("not asked again (asked %v)", asked)
}
readable = true // the forge's account reads the repository now
var got TakenIn
switch tc.trigger {
case "the indexer starts again":
if err := ix.IndexRegisteredAll(ctx); err != nil {
t.Fatal(err)
}
case "a take-in of another repository":
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait() // read again in the background
case "a person's take-in":
got = ix.TakeIn(ctx, "snake")
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
t.Errorf("take-in answered %+v", got)
}
}
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
}
// Taken in, it is read no more: a further event asks the forge nothing.
n := len(asked)
if err := applyS(ix, ctx, 3, srcT{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
t.Fatal(err)
}
ix.retryWG.Wait()
if len(asked) != n {
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
}
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
}
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
t.Errorf("a take-in of an unregistered module answered %+v", none)
}
})
}
}
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
const repo = "http://novox.internal:20000/novox/photos.git"
b := NewBuilds()
for _, r := range [][2]any{
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
} {
_ = b.Put(r[1].([]byte))
}
got := b.RegistrationsOf("photos")
if len(got) != 1 || got[0].Ref != "nox-mesh" {
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
}
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
_, _ = k, b.Put(v) // a trunk's ref beats a branch of its own
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
t.Fatalf("registrations = %+v", got)
}
// An entry of another kind in the builder's log (the old reference form's close, design 51), of this form or another,
// is no build: passed over, never refused.
for _, other := range []string{
fmt.Sprintf(`{"v":%d,"event":"old-form-closed","module":"node-engine","at":"2026-10-04T09:00:00Z"}`, keys.Version),
`{"v":99,"event":"old-form-closed","at":"2026-10-04T09:00:00Z","nodes":["ace"]}`,
} {
if err := b.Put([]byte(other)); err != nil {
t.Errorf("an entry of another kind is refused: %v", err)
}
}
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
t.Fatalf("after entries of another kind, registrations = %+v", got)
}
}
@@ -17,7 +17,7 @@ import (
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
// indexer by an ask in takeins, as every hand act does, since the indexer's process serves no request subject.
const (
CatalogRevisionTool = "indexer_catalog_revision"
CatalogRevisionsTool = "indexer_catalog_revisions"
@@ -7,33 +7,24 @@ import (
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
// of the indexer's states stay in mesh-delivery-view.
// The indexer's hand acts (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): the tools bundle
// serves them through the tool runner; the indexer's process serves no request subject (ADR 0437 §4), so a tool writes
// an ask into the indexer's state takeins, which the indexer watches and answers there, as the builder answers
// builder_ask_again. No tool takes a head in by hand: the gate's gate_head_take is the one hand take of a head (design 47
// H §742, H-A-105).
const (
TakeInTool = "indexer_take_in"
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
// TakeInsState is the indexer's state of hand-act asks: the bucket mesh-delivery-indexer_takeins.
TakeInsState = "takeins"
)
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
// indexer read it, its answer.
// TakeInAsk is a value of the takeins state: which hand act a person asked, why, who and when, and, once the indexer read
// it, its answer.
type TakeInAsk struct {
V int `json:"v"`
Module string `json:"module,omitempty"`
// Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module.
Repository string `json:"repository,omitempty"`
Why string `json:"why,omitempty"`
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
CatchUp bool `json:"catch-up,omitempty"`
Why string `json:"why,omitempty"`
// SeatDefinitions asks a contracts module's trunk seat definitions to be read again now (indexer_seat_definitions_again,
// DLV-I19).
SeatDefinitions bool `json:"seat-definitions,omitempty"`
@@ -91,51 +82,12 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
if a.V != 1 {
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
}
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil {
if a.Module == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil {
return a, errors.New("a takeins value naming no module, repository, pending wait or merge")
}
return a, nil
}
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
module = strings.TrimSpace(module)
if module == "" {
return "", nil, errors.New("name the module to take in (module)")
}
key, err := meshcatalogclient.ModulesKey(module)
if err != nil {
return "", nil, err
}
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// NewTakeInRepository is the value indexer_take_in writes for a repository: its key and the value.
func NewTakeInRepository(repository, why, by string, at time.Time) (string, []byte, error) {
repository = strings.TrimSpace(repository)
key, err := trunk.RepositoryKey(repository)
if err != nil {
return "", nil, err
}
data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "repository." + key, data, err
}
// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first.
const CatchUpKey = "catch-up"
// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every
// repository the gate's source names (CatchUp).
func NewCatchUp(why, by string, at time.Time) (string, []byte, error) {
why = strings.TrimSpace(why)
if why == "" {
why = "the catch-up after a restore"
}
data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()})
return CatchUpKey, data, err
}
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
const TakeInMergeTool = "indexer_take_in_merge"
@@ -0,0 +1,33 @@
package index
import "encoding/json"
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
type RepoRead struct {
Root string
Trees map[string]string
Manifests map[string]string
Locks map[string]map[string]string
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
Packages map[string][]string
}
// TakenIn is the answer of an ask of the state takeins: whether what was asked is done, of which module, from where, and
// otherwise why not.
type TakenIn struct {
Module string `json:"module"`
Stored bool `json:"stored"`
Held bool `json:"held,omitempty"`
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Path string `json:"path,omitempty"`
Answer string `json:"answer"`
// Handed and Dropped are what a hand act on the catalog revisions moved and removed (revision_tools.go).
Handed []string `json:"handed,omitempty"`
Dropped []string `json:"dropped,omitempty"`
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
// Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON.
Registry json.RawMessage `json:"registry,omitempty"`
}
@@ -140,11 +140,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("state = %+v, want %+v", m.State, want)
}
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the
// catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves.
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1. Design 47 H
// §742 (H-A-105): no catch-up tool of the indexer's follows a restore, indexer_take_in having gone with the builder-log
// index; the gate's gate_head_take is the one hand take of a head.
if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 ||
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) {
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool)
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != "" {
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, no catch-up tool", m.Objects, index.CatalogRevisionsObjects)
}
// D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
@@ -185,11 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// 1945: the hand acts are served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules
// kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
want := []string{"indexer_take_in", "indexer_old_sections"}
want := []string{"indexer_old_sections"}
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
want = append(want, v.Tool)
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
@@ -204,7 +205,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") {
if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -224,7 +225,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("the indexer declares the state %s, which design 47 G retires", s.Name)
}
}
for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads"} {
for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads", "indexer_take_in"} {
if slices.Contains(m.Tools, retired) {
t.Errorf("the indexer serves %s, which design 47 G retires", retired)
}
+1 -2
View File
@@ -27,7 +27,6 @@
}
],
"tools": [
"indexer_take_in",
"indexer_old_sections",
"manifests",
"modules",
@@ -125,7 +124,7 @@
{ "name": "mesh-delivery-indexer-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
],
"objects": [
{ "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" }
{ "name": "catalog-revisions", "max-mib": 512, "form": 1 }
],
"emits": [
"take-in.pending",