H-A-48, H-A-105: the builder-log index and indexer_take_in go
TakeIn, TakeInRepository, the catch-up, IndexRegistered and RetryUnread go: a module is taken in only from a source revision, and the gate's gate_head_take is the one hand take of a head. The manifest serves no indexer_take_in and names no catch-up tool beside the catalog revisions. Builds: design 47 H §742, §764 (the indexer); H-A-48, H-A-105 (a) Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's tools (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. A hand act writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4); indexer_judge asks the same way, and
|
||||
// indexer_verdicts reads the indexer's verdicts state. The read tools of the indexer's states are
|
||||
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||
@@ -47,48 +47,6 @@ func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCoun
|
||||
}
|
||||
return map[string]any{"sections": c.Counts, "reads": c.Reads}, nil
|
||||
}},
|
||||
{Name: index.TakeInTool,
|
||||
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
|
||||
"repository the forge could not read (a 404 before the forge's account could read it). A module the indexer holds " +
|
||||
"whose record stands behind the head the gate's source names (a merge whose merged message never reached the " +
|
||||
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
|
||||
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||
"indexer_module (mesh-delivery-view) reads the result. With repository instead of module it takes in the head the gate's " +
|
||||
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
|
||||
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
|
||||
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
|
||||
"that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " +
|
||||
"catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " +
|
||||
"every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " +
|
||||
"after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
var key string
|
||||
var value []byte
|
||||
var err error
|
||||
switch {
|
||||
case str(args, "repository") == "" && str(args, "module") == "":
|
||||
key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now())
|
||||
case str(args, "repository") != "" && str(args, "module") == "":
|
||||
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
default:
|
||||
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.SeatDefinitionsAgainTool,
|
||||
Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " +
|
||||
"holds the module at, and keep them in the seat-definitions state as a take-in would (seat-definition-changed said where a digest moved; the trunk's seat-definition-unparsed and " +
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8). Its
|
||||
// tool indexer_take_in (the program mesh-delivery-indexer-tools) asks through its state takeins, which it watches.
|
||||
// hand-act tools (the program mesh-delivery-indexer-tools) ask through its state takeins, which it watches.
|
||||
//
|
||||
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
||||
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
||||
@@ -384,8 +384,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
|
||||
}
|
||||
done := step("reading the build log")
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
// The log's entries at the start are read before any fingerprint is computed (below).
|
||||
var started atomic.Bool
|
||||
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
|
||||
if err := calls.ReadLog(builds.BuilderModule, builds.BuilderLog).Watch("", stdio.LogFromAll, func(le stdio.LogEntry) error {
|
||||
@@ -394,9 +393,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
}
|
||||
if started.Load() {
|
||||
if err := ix.IndexRegisteredModule(ctx, le.Key); err != nil {
|
||||
logger.Printf("the registered module %s is not indexed: %v", le.Key, err)
|
||||
}
|
||||
// A trunk build's generated files out of step with their seat definitions (DLV-I20).
|
||||
ix.ApplyOutOfStepBuild(ctx, le.Value)
|
||||
// A done build's imports are kept on its module (ADR 0593 §2); at the start they are kept once the packages are.
|
||||
@@ -497,21 +493,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
done()
|
||||
done = step("indexing the registered modules")
|
||||
// A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names
|
||||
// it and the bucket holds none (registered.go); the entries that follow are heard above.
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
logger.Printf("indexing the registered modules at start: %v", err)
|
||||
}
|
||||
done()
|
||||
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||
// merged message (index.RetryUnread).
|
||||
// A person's hand act writes the takeins state (issue 1945): watched, as every other input.
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) }
|
||||
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
|
||||
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again")
|
||||
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "no hand act of the indexer is answered until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, tw, eachTakeIn) }()
|
||||
}
|
||||
|
||||
@@ -13,8 +13,6 @@ import (
|
||||
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
SeatDefinitionsAgain(ctx context.Context, module string) index.TakenIn
|
||||
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
|
||||
RecomputeByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
@@ -28,8 +26,8 @@ type takerIn interface {
|
||||
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
// module in now and is written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and
|
||||
// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
@@ -74,13 +72,10 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
a.Module = r.Module
|
||||
} else if a.Fingerprints {
|
||||
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
|
||||
} else if a.Repository != "" {
|
||||
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
|
||||
a.Module = a.Repository
|
||||
} else if a.SeatDefinitions {
|
||||
asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module)
|
||||
} else {
|
||||
r = ix.TakeIn(ctx, a.Module)
|
||||
r = index.TakenIn{Module: a.Module, Answer: "no hand act is named: the ask is left unanswered by any act"}
|
||||
}
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a %s asked by %s (%s): %s", a.Module, asked, a.By, a.Why, r.Answer)
|
||||
|
||||
@@ -2,8 +2,6 @@ package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -15,7 +13,7 @@ import (
|
||||
|
||||
// Builds holds the builder's log (mesh-delivery-builder.build-log, novox/hq ADR 0571 clause 1) for what the indexer reads
|
||||
// of past builds: the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no
|
||||
// fingerprint, and the places a module registered from its own repository was built from (RegistrationsOf). It is filled
|
||||
// fingerprint. It is filled
|
||||
// from the log's entries at the indexer's start (every one the log keeps, 30 days) and then each new one as the builder
|
||||
// appends it; no entry is ever taken out, since only age takes one from the log.
|
||||
//
|
||||
@@ -64,6 +62,11 @@ func newer(a, b buildrecord.BuildRecord) bool {
|
||||
return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID)
|
||||
}
|
||||
|
||||
// succeeded is whether the record is an outcome that did not fail.
|
||||
func succeeded(r buildrecord.BuildRecord) bool {
|
||||
return r.State == "" && r.Failed == ""
|
||||
}
|
||||
|
||||
// PathOf is the directory of a module of the repository as its newest build record of that repository names it, its
|
||||
// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository
|
||||
// names the module (issue 1614).
|
||||
@@ -76,7 +79,7 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
|
||||
if r.Module != module || !trunk.SameRepository(r.Repository, repository) {
|
||||
continue
|
||||
}
|
||||
t := registered(r) && trunkLike[r.Ref]
|
||||
t := succeeded(r) && trunkLike[r.Ref]
|
||||
if best == nil || (t && !onTrunk) || (t == onTrunk && newer(r, *best)) {
|
||||
c := r
|
||||
best, onTrunk = &c, t
|
||||
@@ -87,101 +90,3 @@ func (b *Builds) PathOf(module, repository string) (string, bool) {
|
||||
}
|
||||
return pathOf(strings.Trim(best.Path, "/")), true
|
||||
}
|
||||
|
||||
// Registration is a module registered in the catalogue from its own repository, as the builder log.s newest end of a build
|
||||
// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was
|
||||
// built at.
|
||||
type Registration struct {
|
||||
Module, Repository, Path, Commit, Ref string
|
||||
}
|
||||
|
||||
// registered is whether the record is an outcome that did not fail: what registers a module. It asks for no particular branch: a
|
||||
// repository's trunk is named main, master, or what its owner chose (the registered photos, de-spiegel and link2pay were
|
||||
// built from nox-mesh, and every record of them names that ref).
|
||||
func registered(r buildrecord.BuildRecord) bool {
|
||||
return r.State == "" && r.Failed == ""
|
||||
}
|
||||
|
||||
// RegistrationsOf are the places a module was registered from, by its outcome records: the newest record of each
|
||||
// repository and directory (a module's name was once another repository's, as photos was a directory of the catalogue's).
|
||||
// Records of a trunk's ref (main, master, trunk, none) come before those of another branch, then the newer first, so the
|
||||
// first that holds the module's manifest is the one indexed.
|
||||
func (b *Builds) RegistrationsOf(module string) []Registration {
|
||||
type place struct{ repository, path string }
|
||||
byPlace := map[place][]buildrecord.BuildRecord{}
|
||||
b.mu.Lock()
|
||||
for _, r := range b.records {
|
||||
if r.Module == module && registered(r) {
|
||||
k := place{trunk.RepositoryName(r.Repository), pathOf(strings.Trim(r.Path, "/"))}
|
||||
byPlace[k] = append(byPlace[k], r)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
// Within one place the record is that of the ref a registration comes from, never a newer pull request's or try's: a
|
||||
// trunk's ref (main, master, trunk, none), else the ref most of the place's records name (the registered photos, de-spiegel
|
||||
// and link2pay were built from nox-mesh), the newer of two equal; then its newest record.
|
||||
best := map[place]buildrecord.BuildRecord{}
|
||||
for k, recs := range byPlace {
|
||||
count, latest := map[string]int{}, map[string]buildrecord.BuildRecord{}
|
||||
for _, r := range recs {
|
||||
count[r.Ref]++
|
||||
if cur, ok := latest[r.Ref]; !ok || newer(r, cur) {
|
||||
latest[r.Ref] = r
|
||||
}
|
||||
}
|
||||
var pick string
|
||||
first := true
|
||||
for ref, rec := range latest {
|
||||
switch {
|
||||
case first:
|
||||
case trunkLike[ref] != trunkLike[pick]:
|
||||
if !trunkLike[ref] {
|
||||
continue
|
||||
}
|
||||
case count[ref] != count[pick]:
|
||||
if count[ref] < count[pick] {
|
||||
continue
|
||||
}
|
||||
case !newer(rec, latest[pick]):
|
||||
continue
|
||||
}
|
||||
pick, first = ref, false
|
||||
}
|
||||
best[k] = latest[pick]
|
||||
}
|
||||
recs := slices.Collect(maps.Values(best))
|
||||
slices.SortFunc(recs, func(x, y buildrecord.BuildRecord) int {
|
||||
switch {
|
||||
case trunkLike[x.Ref] != trunkLike[y.Ref]:
|
||||
if trunkLike[x.Ref] {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
case newer(x, y):
|
||||
return -1
|
||||
case newer(y, x):
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(x.Repository+x.Path, y.Repository+y.Path)
|
||||
})
|
||||
out := make([]Registration, 0, len(recs))
|
||||
for _, r := range recs {
|
||||
out = append(out, Registration{Module: r.Module, Repository: r.Repository, Path: r.Path, Commit: r.Commit, Ref: r.Ref})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Registered is the registrations of every module a record names, by module name.
|
||||
func (b *Builds) Registered() []Registration {
|
||||
b.mu.Lock()
|
||||
names := map[string]bool{}
|
||||
for _, r := range b.records {
|
||||
names[r.Module] = true
|
||||
}
|
||||
b.mu.Unlock()
|
||||
var out []Registration
|
||||
for _, name := range slices.Sorted(maps.Keys(names)) {
|
||||
out = append(out, b.RegistrationsOf(name)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -148,12 +148,7 @@ type Indexer struct {
|
||||
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]whole // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
@@ -234,7 +229,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f whole) (Applied, error) {
|
||||
return applied, err
|
||||
}
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a take-in, in the background (issue 1945)
|
||||
lateMerge = w.late != ""
|
||||
}
|
||||
// The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's.
|
||||
|
||||
@@ -467,13 +467,9 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
repository string // modules.a's repository at the end, "" for none
|
||||
deletedKeyA bool
|
||||
// Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge
|
||||
// reads at the head of `to`; takeIn is a repository taken in after the merges (indexer_take_in), whose answer must
|
||||
// hold answer and be stored or not.
|
||||
// reads at the head of `to`.
|
||||
heads bool
|
||||
trunk []string
|
||||
takeIn string
|
||||
answer string
|
||||
stored bool
|
||||
bGoneAtEnd bool
|
||||
}{
|
||||
{name: "named while its trunk holds it: left", merges: []whole{held, named}, repository: from},
|
||||
@@ -484,12 +480,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
merges: []whole{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
|
||||
{name: "no module.json left, no other trunk holds it: deleted",
|
||||
merges: []whole{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with no module.json: handed over from the other trunk",
|
||||
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with modules: answered by name",
|
||||
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
|
||||
{name: "taken in by repository, no head named: answered so",
|
||||
merges: []whole{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mods := &deletesSeen{memStore: newMem()}
|
||||
@@ -502,9 +492,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
if repository == to && sha == toHead {
|
||||
return tc.trunk, nil
|
||||
}
|
||||
if repository == from && tc.takeIn == from {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
ix.Read = func(_ context.Context, repository, sha, dir string) (RepoRead, error) {
|
||||
@@ -519,12 +506,6 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.takeIn != "" {
|
||||
got := ix.TakeInRepository(context.Background(), tc.takeIn)
|
||||
if got.Stored != tc.stored || !strings.Contains(got.Answer, tc.answer) {
|
||||
t.Errorf("take-in answered %+v, want stored %v and %q", got, tc.stored, tc.answer)
|
||||
}
|
||||
}
|
||||
m, ok := moduleOf(t, mods.memStore, "a")
|
||||
if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository {
|
||||
t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs)
|
||||
|
||||
@@ -1,461 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
|
||||
// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge
|
||||
// followed the registration. The controller's outcome records name each registered module's repository, directory and
|
||||
// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the
|
||||
// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged
|
||||
// message as for every other repository.
|
||||
|
||||
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
|
||||
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
|
||||
type RepoRead struct {
|
||||
Root string
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
Packages map[string][]string
|
||||
}
|
||||
|
||||
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
|
||||
// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The
|
||||
// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or
|
||||
// names another module is said and left. A forge that could not say (a timeout, or a 404 while the forge's account could
|
||||
// not read the repository) is an error, and the module is kept as unread: it is read again at the next source revision
|
||||
// and the next merged message (RetryUnread), not only when the indexer starts again (issue 1945).
|
||||
func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error {
|
||||
_, err := ix.indexRegistered(ctx, r, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// taken is what indexRegistered did with one registration: the module written, held already, or the reason it was not.
|
||||
type taken struct {
|
||||
stored, held bool
|
||||
repository, commit, dir string
|
||||
why string
|
||||
}
|
||||
|
||||
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
|
||||
return ix.indexAt(ctx, r, force, false)
|
||||
}
|
||||
|
||||
// indexAt is indexRegistered; reread also reads a module the modules state holds (a person's take-in of a held module
|
||||
// whose record stands behind the head the gate's source names, issue 1126).
|
||||
func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bool) (taken, error) {
|
||||
if ix.Modules == nil || ix.Read == nil {
|
||||
return taken{why: "the indexer reads no forge or writes no modules"}, nil
|
||||
}
|
||||
key, err := meshcatalogclient.ModulesKey(r.Module)
|
||||
if err != nil {
|
||||
ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err)
|
||||
return taken{why: fmt.Sprintf("the module has no usable key: %v", err)}, nil
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if found && !reread {
|
||||
ix.markRead(r.Module)
|
||||
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
|
||||
}
|
||||
repository := trunk.RepositoryName(r.Repository)
|
||||
commit := r.Commit
|
||||
dir := strings.Trim(r.Path, "/")
|
||||
if dir == meshcatalogclient.RootPath {
|
||||
dir = ""
|
||||
}
|
||||
// The head the gate's source names for the repository, when it names one, else the registration's commit. A module a
|
||||
// merge removed is absent at the head and is said and left.
|
||||
ix.mu.Lock()
|
||||
for repo, head := range ix.source.heads {
|
||||
if trunk.SameRepository(repo, r.Repository) && head != "" {
|
||||
commit = head
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
t := taken{repository: repository, commit: commit, dir: dir}
|
||||
if repository == "" || commit == "" {
|
||||
ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module)
|
||||
t.why = "the controller's record names no repository or commit"
|
||||
return t, nil
|
||||
}
|
||||
// One try for each repository, commit and directory: a repository with no manifest where it was registered is said once,
|
||||
// not at every record of the controller's builds. A person's take-in (force) reads again.
|
||||
tried := repository + "@" + commit + ":" + dir
|
||||
ix.mu.Lock()
|
||||
if ix.tried[tried] && !force {
|
||||
ix.mu.Unlock()
|
||||
t.why = "tried already at this commit and directory"
|
||||
return t, nil
|
||||
}
|
||||
if ix.tried == nil {
|
||||
ix.tried = map[string]bool{}
|
||||
}
|
||||
ix.tried[tried] = true
|
||||
ix.mu.Unlock()
|
||||
read, err := ix.Read(ctx, repository, commit, dir)
|
||||
if err != nil {
|
||||
ix.mu.Lock()
|
||||
delete(ix.tried, tried) // a forge that could not say is asked again
|
||||
if ix.unread == nil {
|
||||
ix.unread = map[string]bool{}
|
||||
}
|
||||
ix.unread[r.Module] = true // and read again on the next source revision or merged message (RetryUnread)
|
||||
ix.mu.Unlock()
|
||||
return t, fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err)
|
||||
}
|
||||
ix.markRead(r.Module)
|
||||
raw, ok := read.Manifests[dir]
|
||||
if !ok || read.Trees[dir] == "" {
|
||||
ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir)
|
||||
t.why = fmt.Sprintf("%s at %s holds no module.json at %q", repository, commit, dir)
|
||||
return t, nil
|
||||
}
|
||||
if name := manifestModule(raw); name != r.Module {
|
||||
ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name)
|
||||
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||
return t, nil
|
||||
}
|
||||
f := whole{Repository: repository, Merge: commit, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]},
|
||||
Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
}
|
||||
if p := read.Packages[dir]; len(p) > 0 {
|
||||
f.Packages = map[string][]string{dir: p}
|
||||
}
|
||||
if _, err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
return t, err
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return t, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if !found {
|
||||
t.why = "its manifest was read but the modules state holds no value of it (the indexer's log says why)"
|
||||
return t, nil
|
||||
}
|
||||
t.stored = true
|
||||
ix.logf("%s: the registered module is indexed from %s at %s", r.Module, repository, commit)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// markRead takes a module out of the unread ones: the forge answered for it.
|
||||
func (ix *Indexer) markRead(module string) {
|
||||
ix.mu.Lock()
|
||||
delete(ix.unread, module)
|
||||
ix.mu.Unlock()
|
||||
}
|
||||
|
||||
// RetryUnread reads again every registered module whose last read the forge could not answer (a timeout, or a 404 while
|
||||
// the forge's account could not read the repository; issue 1945). It is called on the events that already reach the
|
||||
// indexer, a new source revision and a merged message, so a repository that became readable is taken in without a hand
|
||||
// act and without a timer of its own (ADR 0437 §8). What still cannot be read is said and kept for the next.
|
||||
func (ix *Indexer) RetryUnread(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
names := slices.Sorted(maps.Keys(ix.unread))
|
||||
ix.mu.Unlock()
|
||||
for _, module := range names {
|
||||
if err := ix.IndexRegisteredModule(ctx, module); err != nil {
|
||||
ix.logf("the registered module %s is still not indexed: %v", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kickRetry runs RetryUnread in one background goroutine, so the message or revision that asked it is acknowledged without
|
||||
// waiting on the forge (each unread module costs up to two forge calls of 30 s each). A kick while one runs makes it run
|
||||
// once more when it ends; never two at once.
|
||||
func (ix *Indexer) kickRetry(ctx context.Context) {
|
||||
ix.mu.Lock()
|
||||
none := len(ix.unread) == 0
|
||||
ix.mu.Unlock()
|
||||
if none {
|
||||
return
|
||||
}
|
||||
ix.retryAgain.Store(true)
|
||||
if !ix.retrying.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
ix.retryWG.Add(1)
|
||||
go func() {
|
||||
defer ix.retryWG.Done()
|
||||
for ix.retryAgain.Swap(false) {
|
||||
ix.RetryUnread(ctx)
|
||||
}
|
||||
ix.retrying.Store(false)
|
||||
// A kick between the last Swap and the Store found it running and returned: run for it.
|
||||
if ix.retryAgain.Load() {
|
||||
ix.kickRetry(ctx)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// TakenIn is the answer of a take-in (the tool indexer_take_in): whether the module is now in the modules state, from
|
||||
// where, and otherwise why not.
|
||||
type TakenIn struct {
|
||||
Module string `json:"module"`
|
||||
Stored bool `json:"stored"`
|
||||
Held bool `json:"held,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Answer string `json:"answer"`
|
||||
// Handed and Dropped are the modules a take-in of a repository (TakeInRepository) moved to another repository's trunk
|
||||
// and deleted.
|
||||
Handed []string `json:"handed,omitempty"`
|
||||
Dropped []string `json:"dropped,omitempty"`
|
||||
// Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
|
||||
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
|
||||
// Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON.
|
||||
Registry json.RawMessage `json:"registry,omitempty"`
|
||||
}
|
||||
|
||||
// TakeIn reads a registered module's manifest now, whatever was tried before, and writes it as a merge would (the tool
|
||||
// indexer_take_in, issue 1945). A module the modules state holds is answered so and left; one with no registration in the
|
||||
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
|
||||
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
if again, ok := ix.takeInHeld(ctx, module); ok {
|
||||
return again
|
||||
}
|
||||
var regs []Registration
|
||||
if ix.Builds != nil {
|
||||
regs = ix.Builds.RegistrationsOf(module)
|
||||
}
|
||||
if len(regs) == 0 {
|
||||
if r, why, ok := ix.registeredAtBranch(ctx, module); ok {
|
||||
regs = []Registration{r}
|
||||
} else if why != "" {
|
||||
ix.logf("take-in of %s: %s", module, why)
|
||||
out.Answer = why
|
||||
return out
|
||||
}
|
||||
}
|
||||
if len(regs) == 0 {
|
||||
if key, err := meshcatalogclient.ModulesKey(module); err == nil && ix.Modules != nil {
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
|
||||
out.Held, out.Answer = true, "the modules state holds it already: a merge to its repository writes it from now on"
|
||||
return out
|
||||
}
|
||||
}
|
||||
out.Answer = "the controller's build records name no registration of " + module + ": a module from outside the indexed repositories is known only from them, and one inside them from a merge"
|
||||
return out
|
||||
}
|
||||
var whys []string
|
||||
for _, r := range regs {
|
||||
t, err := ix.indexRegistered(ctx, r, true)
|
||||
out.Repository, out.Commit, out.Path = t.repository, t.commit, pathOf(t.dir)
|
||||
switch {
|
||||
case err != nil:
|
||||
whys = append(whys, err.Error())
|
||||
continue
|
||||
case t.held:
|
||||
out.Held, out.Answer = true, t.why
|
||||
out.Repository, out.Commit, out.Path = "", "", ""
|
||||
return out
|
||||
case t.stored:
|
||||
out.Stored = true
|
||||
out.Answer = fmt.Sprintf("written to the modules state from %s at %s", t.repository, t.commit)
|
||||
return out
|
||||
}
|
||||
whys = append(whys, t.why)
|
||||
}
|
||||
out.Answer = "not taken in: " + strings.Join(whys, "; ")
|
||||
return out
|
||||
}
|
||||
|
||||
// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one.
|
||||
func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.RegistrationsOf(module) {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is
|
||||
// said and the others go on; the first error is answered.
|
||||
func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
|
||||
if ix.Builds == nil {
|
||||
return nil
|
||||
}
|
||||
var first error
|
||||
for _, r := range ix.Builds.Registered() {
|
||||
if err := ix.IndexRegistered(ctx, r); err != nil {
|
||||
ix.logf("the registered module %s is not indexed: %v", r.Module, err)
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// takeInHeld reads again, at the head the gate's source names for its repository, a module the modules state holds whose
|
||||
// record stands at another commit, and writes it as a merge would (only when it changed). It is the recovery for a merge
|
||||
// whose merged message never reached the indexer (issue 1126: mesh-catalog#535's message was given up by the forge bridge
|
||||
// after its reads timed out, and mesh-vault's record stayed at the merge before). It answers false where it does not
|
||||
// apply: no such module held, or no head named, or the record at that head already.
|
||||
func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) {
|
||||
out := TakenIn{Module: module}
|
||||
key, err := meshcatalogclient.ModulesKey(module)
|
||||
if err != nil || ix.Modules == nil {
|
||||
return out, false
|
||||
}
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return out, false
|
||||
}
|
||||
cur, err := meshcatalogclient.DecodeModule(value)
|
||||
if err != nil || cur.Repository == "" || cur.Path == "" {
|
||||
return out, false
|
||||
}
|
||||
head := ""
|
||||
ix.mu.Lock()
|
||||
for repo, h := range ix.source.heads {
|
||||
if trunk.SameRepository(repo, cur.Repository) && h != "" {
|
||||
head = h
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
if head == "" || head == cur.Commit {
|
||||
return out, false
|
||||
}
|
||||
// Never backwards (order.go): a source head older than the record (a merge applied before the source moved) is never
|
||||
// read and written back over it. A diverged or rewritten history, or a record whose commit the forge no longer has, is
|
||||
// not older and is read; where the forge cannot say otherwise, nothing is read. One forge call.
|
||||
old, why := true, "the forge cannot be asked"
|
||||
if ix.Contains != nil {
|
||||
var err error
|
||||
if old, err = ix.older(ctx, cur.Repository, head, cur.Commit); err != nil {
|
||||
old, why = true, err.Error()
|
||||
} else if old {
|
||||
why = fmt.Sprintf("%.12s is older than its record at %.12s", head, cur.Commit)
|
||||
}
|
||||
}
|
||||
if old {
|
||||
out.Held, out.Repository, out.Commit, out.Path = true, cur.Repository, cur.Commit, cur.Path
|
||||
out.Answer = fmt.Sprintf("held; the source names another head, %.12s, and not one newer than its record (%s), so it is not read again", head, why)
|
||||
return out, true
|
||||
}
|
||||
t, err := ix.indexAt(ctx, Registration{Module: module, Repository: cur.Repository, Path: cur.Path, Commit: head}, true, true)
|
||||
out.Held, out.Repository, out.Commit, out.Path = true, t.repository, t.commit, pathOf(t.dir)
|
||||
switch {
|
||||
case err != nil:
|
||||
out.Answer = "held, and not read again at the source's head: " + err.Error()
|
||||
case !t.stored:
|
||||
out.Answer = "held, and not read again at the source's head: " + t.why
|
||||
default:
|
||||
after, _, _, _ := ix.Modules.Get(ctx, key)
|
||||
out.Stored = true
|
||||
if string(after) == string(value) {
|
||||
out.Answer = fmt.Sprintf("held, and read again from %s at %s: unchanged there, so its record stands", t.repository, t.commit)
|
||||
} else {
|
||||
out.Answer = fmt.Sprintf("held, read again from %s at %s and written to the modules state: it changed since %.12s", t.repository, t.commit, cur.Commit)
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// TakeInRepository takes in the head the gate's source names for a repository whose merged message the indexer never
|
||||
// heard (the tool indexer_take_in with repository, issue 1367). A head whose trunk holds no module.json is applied as
|
||||
// the merged message of a merge that leaves none: every module of the repository goes, one another trunk holds handed over
|
||||
// from there first. A head whose trunk holds modules is answered "by name": each is taken in with its own name.
|
||||
func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) TakenIn {
|
||||
out := TakenIn{Repository: repository}
|
||||
ix.mu.Lock()
|
||||
head := ""
|
||||
for repo, h := range ix.source.heads {
|
||||
if trunk.SameRepository(repo, repository) && h != "" {
|
||||
head, out.Repository = h, repo
|
||||
}
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
if head == "" || ix.Modules == nil || ix.Trunk == nil {
|
||||
out.Answer = "the gate's source names no head of " + repository + ", or the indexer reads no forge or writes no modules"
|
||||
return out
|
||||
}
|
||||
out.Commit = head
|
||||
dirs, err := ix.Trunk(ctx, out.Repository, head)
|
||||
if err != nil {
|
||||
out.Answer = fmt.Sprintf("the modules of %s at %.12s cannot be read, so nothing is taken in: %v", repository, head, err)
|
||||
return out
|
||||
}
|
||||
if len(dirs) > 0 {
|
||||
out.Answer = fmt.Sprintf("%s at %.12s holds modules (%s): take each module in by name (module)", repository, head, strings.Join(dirs, ", "))
|
||||
return out
|
||||
}
|
||||
before, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
f := whole{Repository: out.Repository, Merge: head}
|
||||
if _, err := ix.ApplyMessage(ctx, f); err != nil {
|
||||
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
|
||||
return out
|
||||
}
|
||||
after, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(before)) {
|
||||
if !trunk.SameRepository(before[key].Repository, repository) {
|
||||
continue
|
||||
}
|
||||
if now, ok := after[key]; ok {
|
||||
out.Handed = append(out.Handed, key+" to "+now.Repository+" at "+now.Commit)
|
||||
} else {
|
||||
out.Dropped = append(out.Dropped, key)
|
||||
}
|
||||
}
|
||||
out.Stored = true
|
||||
out.Answer = fmt.Sprintf("%s at %.12s holds no module.json: taken in as a merge that leaves none; handed over %v, dropped %v", repository, head, out.Handed, out.Dropped)
|
||||
return out
|
||||
}
|
||||
|
||||
// registeredAtBranch is the module's registration (the seat mesh-catalog's, ADR 0666) read at the head of its first watched
|
||||
// branch, which is its trunk (design 47 G A-85): a module from outside the indexed repositories, watched on main, master or
|
||||
// a branch its owner chose (nox-mesh), is taken in from there when the controller's build records name none. why says what
|
||||
// stopped it; none and not ok is no registration.
|
||||
func (ix *Indexer) registeredAtBranch(ctx context.Context, module string) (Registration, string, bool) {
|
||||
if ix.Registry == nil || ix.Registry.Registrations == nil || ix.BranchHead == nil {
|
||||
return Registration{}, "", false
|
||||
}
|
||||
reg, found, err := ix.Registry.Registration(ctx, module)
|
||||
if err != nil {
|
||||
return Registration{}, fmt.Sprintf("the registration of %s cannot be read: %v", module, err), false
|
||||
}
|
||||
if !found {
|
||||
return Registration{}, "", false
|
||||
}
|
||||
if len(reg.Branches) == 0 || reg.Origin == "" {
|
||||
return Registration{}, fmt.Sprintf("the registration of %s names no repository or watched branch", module), false
|
||||
}
|
||||
branch := reg.Branches[0]
|
||||
head, err := ix.BranchHead(ctx, reg.Origin, branch)
|
||||
if err != nil || head == "" {
|
||||
return Registration{}, fmt.Sprintf("the head of %s's branch %s cannot be read: %v", reg.Origin, branch, err), false
|
||||
}
|
||||
ix.logf("take-in of %s: registered from %s, read at the head %s of its first watched branch %s", module, reg.Origin, head, branch)
|
||||
return Registration{Module: module, Repository: reg.Origin, Path: path.Dir(reg.Path), Commit: head, Ref: branch}, "", true
|
||||
}
|
||||
@@ -1,326 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here
|
||||
const forge = "http://novox.internal:20000/jschoubben/snake.git"
|
||||
snake := `{"module":"snake","version":"1.0"}`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
read RepoRead
|
||||
err error
|
||||
ref string // the build record's ref
|
||||
heads map[string]string // the gate source's heads
|
||||
wantRead bool // the forge is read
|
||||
held bool // modules.snake exists already
|
||||
other bool // modules.other of the same repository exists
|
||||
want bool // modules.snake is written
|
||||
}{
|
||||
{wantRead: true, name: "a module at the root of its repository", path: "", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a root spelled as a dot", path: ".", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a module in a directory", path: "modules/snake", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}},
|
||||
{wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{name: "a module the bucket holds is left", path: "", held: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "no manifest at the path", path: "modules/x",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a manifest of another module", path: "",
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1.0"}`}}},
|
||||
{wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository built from a branch of its own (nox-mesh)", path: "", ref: "nox-mesh", want: true,
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2},
|
||||
read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}},
|
||||
{wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", ""))
|
||||
if tc.held {
|
||||
if err := mods.Create(context.Background(), "snake", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.other {
|
||||
if err := mods.Create(context.Background(), "other", mustModule(t, meshcatalog.ModulesByModuleValue{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ix.Manifests = newMem()
|
||||
var asked struct {
|
||||
repo, sha, dir string
|
||||
n int
|
||||
}
|
||||
ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) {
|
||||
asked.repo, asked.sha, asked.dir = repo, sha, dir
|
||||
asked.n++
|
||||
return tc.read, tc.err
|
||||
}
|
||||
if tc.heads != nil {
|
||||
ix.source = sourceNow{rev: 1, heads: tc.heads}
|
||||
}
|
||||
err := ix.IndexRegisteredAll(context.Background())
|
||||
if (err != nil) != (tc.err != nil) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's
|
||||
if tc.heads != nil {
|
||||
wantCommit = h2 // the source's head
|
||||
}
|
||||
if tc.wantRead && asked.sha != wantCommit {
|
||||
t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit)
|
||||
}
|
||||
wantReads := 1
|
||||
if !tc.wantRead {
|
||||
wantReads = 0
|
||||
}
|
||||
if tc.wantRead && asked.dir != dirOf(tc.path) {
|
||||
t.Errorf("the forge was asked for directory %q", asked.dir)
|
||||
}
|
||||
if asked.n != wantReads {
|
||||
t.Errorf("forge reads = %d, want %d", asked.n, wantReads)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "snake")
|
||||
if tc.want {
|
||||
if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) {
|
||||
t.Fatalf("modules.snake = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(ix.Manifests.(*memStore).vals) != 1 {
|
||||
t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals))
|
||||
}
|
||||
if asked.repo != "jschoubben/snake" {
|
||||
t.Errorf("the forge was asked for %q", asked.repo)
|
||||
}
|
||||
} else if !tc.held && ok {
|
||||
t.Fatalf("modules.snake written: %+v", m)
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "other"); ok != tc.other {
|
||||
t.Errorf("modules.other held = %v, want %v", ok, tc.other)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 1126: mesh-catalog#535 changed mesh-vault, but its merged message never reached the indexer, so the record
|
||||
// stood at the merge before. A take-in of the held module reads it again at the source's head, only when the record is
|
||||
// behind that head: a head older than the record (a merge applied before the source moved), or one whose order the forge
|
||||
// cannot say, is never written back.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
contains func(context.Context, string, string, string) (bool, error)
|
||||
moved bool
|
||||
}{
|
||||
{name: "a held module behind the source's head is read again", moved: true,
|
||||
contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == h2, nil }},
|
||||
{name: "a source head older than the record is not read", contains: func(_ context.Context, _, commit, in string) (bool, error) { return commit == h2 && in == merge, nil }},
|
||||
// Review of 77d738d: a diverged or rewritten history, or a record whose commit the forge no longer has, is not older.
|
||||
{name: "a source head of a diverged history is read again", moved: true,
|
||||
contains: func(context.Context, string, string, string) (bool, error) { return false, nil }},
|
||||
{name: "a record whose commit the forge no longer has is read again", moved: true, contains: func(context.Context, string, string, string) (bool, error) {
|
||||
return false, fmt.Errorf("seat:git.contains (not-found): the forge holds no commit: %w", ErrNotFound) // the kind, no status in its words
|
||||
}},
|
||||
{name: "a forge that cannot say the order reads nothing", contains: func(context.Context, string, string, string) (bool, error) {
|
||||
return false, errors.New("timeout")
|
||||
}},
|
||||
{name: "no forge to ask reads nothing"},
|
||||
} {
|
||||
t.Run("take-in: "+tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods := modulesIndexer(t)
|
||||
ix.Manifests = newMem()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "treeA"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.source = sourceNow{rev: 2, heads: map[string]string{"novox/mesh-catalog": h2}}
|
||||
ix.Contains = tc.contains
|
||||
var asked []string
|
||||
ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) {
|
||||
asked = append(asked, sha+":"+dir)
|
||||
return RepoRead{Root: "r1", Trees: map[string]string{"modules/a": "treeA2"}, Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}, nil
|
||||
}
|
||||
got := ix.TakeIn(ctx, "a")
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !tc.moved {
|
||||
if got.Stored || !got.Held || len(asked) != 0 || !strings.Contains(got.Answer, "another head") || !ok || m.Commit != merge {
|
||||
t.Errorf("take-in answered %+v, asked %v, modules.a at %.8s", got, asked, m.Commit)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !got.Stored || !got.Held || got.Commit != h2 || got.Path != "modules/a" || !strings.Contains(got.Answer, "written") {
|
||||
t.Errorf("take-in answered %+v", got)
|
||||
}
|
||||
if !ok || m.Commit != h2 || m.Inputs2 == nil || m.Inputs2.Tree != "treeA2" {
|
||||
t.Errorf("modules.a = %+v (%v), want it at %.8s with tree treeA2 (asked %v)", m, ok, h2, asked)
|
||||
}
|
||||
n := len(asked) // at the head now: a take-in answers held and reads nothing
|
||||
if again := ix.TakeIn(ctx, "a"); again.Stored || !again.Held || len(asked) != n {
|
||||
t.Errorf("a take-in of a module at the source's head answered %+v and asked %v", again, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustModule(t *testing.T, m meshcatalog.ModulesByModuleValue) []byte {
|
||||
t.Helper()
|
||||
b, err := meshcatalogclient.EncodeModule(withV(m))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func dirOf(p string) string {
|
||||
if p == "." {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestARegistrationOfAnotherRepositoryDoesNotHideTheRealOne(t *testing.T) { // photos: its newest trunk record was a directory of the catalogue's, then novox/photos at nox-mesh
|
||||
const catalogue, own = "http://novox.internal:20000/novox/mesh-catalog.git", "http://novox.internal:20000/novox/Link2Pay.git"
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "link2pay", own, "", "nox-mesh", "", "2026-10-06T23:02:00Z", ""),
|
||||
rec(t, "build-2", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-04T16:02:00Z", ""),
|
||||
rec(t, "build-3", "link2pay", catalogue, "modules/link2pay", "main", "", "2026-10-03T16:02:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
var asked []string
|
||||
ix.Read = func(_ context.Context, repo, _, dir string) (RepoRead, error) {
|
||||
asked = append(asked, repo+":"+dir)
|
||||
if repo == "novox/Link2Pay" {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"link2pay","version":"1.0"}`}}, nil
|
||||
}
|
||||
return RepoRead{Root: "r0"}, nil
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "link2pay")
|
||||
if !ok || m.Repository != "novox/Link2Pay" {
|
||||
t.Fatalf("modules.link2pay = %+v, found %v", m, ok)
|
||||
}
|
||||
if len(asked) != 2 || asked[0] != "novox/mesh-catalog:modules/link2pay" || asked[1] != "novox/Link2Pay:." && asked[1] != "novox/Link2Pay:" {
|
||||
t.Errorf("the forge was asked %v: want the catalogue's trunk record first, once, then the repository's own", asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { // a timeout or a 404 at the head: no read at an older commit
|
||||
// Issue 1945: car-hunter and snake answered 404 until the forge's account could read them, and were read again only when
|
||||
// the indexer started again. Each row's trigger must now take the module in once the forge reads it.
|
||||
const snake = `{"module":"snake","version":"1.0"}`
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct{ failure, trigger string }{
|
||||
{"timeout", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a take-in of another repository"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
|
||||
{"timeout", "a person's take-in"},
|
||||
} {
|
||||
t.Run(tc.failure+", then "+tc.trigger, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", "http://novox.internal:20000/jschoubben/snake.git", "", "master", "", "2026-10-19T09:00:00Z", ""))
|
||||
ix.Manifests = newMem()
|
||||
ix.source = sourceNow{rev: 1, heads: map[string]string{"jschoubben/snake": h2}}
|
||||
var asked []string
|
||||
readable := false
|
||||
ix.Read = func(_ context.Context, _, sha, _ string) (RepoRead, error) {
|
||||
asked = append(asked, sha)
|
||||
if readable {
|
||||
return RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}, nil
|
||||
}
|
||||
return RepoRead{}, errors.New(tc.failure)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil {
|
||||
t.Errorf("no error")
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "snake"); ok || len(asked) != 1 || asked[0] != h2 {
|
||||
t.Errorf("written %v, asked %v", ok, asked)
|
||||
}
|
||||
if err := ix.IndexRegisteredAll(ctx); err == nil || len(asked) != 2 {
|
||||
t.Errorf("not asked again (asked %v)", asked)
|
||||
}
|
||||
readable = true // the forge's account reads the repository now
|
||||
var got TakenIn
|
||||
switch tc.trigger {
|
||||
case "the indexer starts again":
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "a take-in of another repository":
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a person's take-in":
|
||||
got = ix.TakeIn(ctx, "snake")
|
||||
if !got.Stored || got.Repository != "jschoubben/snake" || got.Commit != h2 || got.Path != "." || got.Answer == "" {
|
||||
t.Errorf("take-in answered %+v", got)
|
||||
}
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "snake"); !ok || m.Manifest == "" || m.Commit != h2 {
|
||||
t.Fatalf("after %s the readable module is not taken in: %+v, found %v (asked %v)", tc.trigger, m, ok, asked)
|
||||
}
|
||||
// Taken in, it is read no more: a further event asks the forge nothing.
|
||||
n := len(asked)
|
||||
if err := applyS(ix, ctx, 3, srcT{Heads: map[string]string{"jschoubben/snake": h2}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait()
|
||||
if len(asked) != n {
|
||||
t.Errorf("the forge was asked again after the module was taken in: %v", asked)
|
||||
}
|
||||
if again := ix.TakeIn(ctx, "snake"); again.Stored || !again.Held || len(asked) != n {
|
||||
t.Errorf("a take-in of a held module answered %+v and asked %v", again, asked)
|
||||
}
|
||||
if none := ix.TakeIn(ctx, "nobody"); none.Stored || none.Held || !strings.Contains(none.Answer, "no registration") {
|
||||
t.Errorf("a take-in of an unregistered module answered %+v", none)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordOfAPullRequestDoesNotReplaceTheRegisteredRef(t *testing.T) { // a newer feat/x record of the same place
|
||||
const repo = "http://novox.internal:20000/novox/photos.git"
|
||||
b := NewBuilds()
|
||||
for _, r := range [][2]any{
|
||||
rec(t, "b1", "photos", repo, "", "nox-mesh", "", "2026-10-01T09:00:00Z", ""),
|
||||
rec(t, "b2", "photos", repo, "", "nox-mesh", "", "2026-10-02T09:00:00Z", ""),
|
||||
rec(t, "b3", "photos", repo, "", "feat/x", "", "2026-10-09T09:00:00Z", ""),
|
||||
} {
|
||||
_ = b.Put(r[1].([]byte))
|
||||
}
|
||||
got := b.RegistrationsOf("photos")
|
||||
if len(got) != 1 || got[0].Ref != "nox-mesh" {
|
||||
t.Fatalf("registrations = %+v, want the ref most records name, not the newer feat/x", got)
|
||||
}
|
||||
k, v := record(t, "b4", "photos", repo, "", "main", "", "2026-10-03T09:00:00Z", "")
|
||||
_, _ = k, b.Put(v) // a trunk's ref beats a branch of its own
|
||||
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
|
||||
t.Fatalf("registrations = %+v", got)
|
||||
}
|
||||
// An entry of another kind in the builder's log (the old reference form's close, design 51), of this form or another,
|
||||
// is no build: passed over, never refused.
|
||||
for _, other := range []string{
|
||||
fmt.Sprintf(`{"v":%d,"event":"old-form-closed","module":"node-engine","at":"2026-10-04T09:00:00Z"}`, keys.Version),
|
||||
`{"v":99,"event":"old-form-closed","at":"2026-10-04T09:00:00Z","nodes":["ace"]}`,
|
||||
} {
|
||||
if err := b.Put([]byte(other)); err != nil {
|
||||
t.Errorf("an entry of another kind is refused: %v", err)
|
||||
}
|
||||
}
|
||||
if got := b.RegistrationsOf("photos"); len(got) != 1 || got[0].Ref != "main" {
|
||||
t.Fatalf("after entries of another kind, registrations = %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
|
||||
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
|
||||
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
|
||||
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
|
||||
// indexer by an ask in takeins, as every hand act does, since the indexer's process serves no request subject.
|
||||
const (
|
||||
CatalogRevisionTool = "indexer_catalog_revision"
|
||||
CatalogRevisionsTool = "indexer_catalog_revisions"
|
||||
|
||||
@@ -7,33 +7,24 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// The indexer's doing tool (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): indexer_take_in
|
||||
// reads a registered module's manifest now and writes it to the modules state. The tools bundle serves it through the tool
|
||||
// runner; the indexer's process serves no request subject (ADR 0437 §4), so the tool writes an ask into the indexer's
|
||||
// state takeins, which the indexer watches and answers there, as the builder answers builder_ask_again. The read tools
|
||||
// of the indexer's states stay in mesh-delivery-view.
|
||||
// The indexer's hand acts (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): the tools bundle
|
||||
// serves them through the tool runner; the indexer's process serves no request subject (ADR 0437 §4), so a tool writes
|
||||
// an ask into the indexer's state takeins, which the indexer watches and answers there, as the builder answers
|
||||
// builder_ask_again. No tool takes a head in by hand: the gate's gate_head_take is the one hand take of a head (design 47
|
||||
// H §742, H-A-105).
|
||||
const (
|
||||
TakeInTool = "indexer_take_in"
|
||||
// TakeInsState is the indexer's state of take-in asks: the bucket mesh-delivery-indexer_takeins, keyed as the modules state.
|
||||
// TakeInsState is the indexer's state of hand-act asks: the bucket mesh-delivery-indexer_takeins.
|
||||
TakeInsState = "takeins"
|
||||
)
|
||||
|
||||
// TakeInAsk is a value of the takeins state: which module a person asked to be taken in, why, who and when, and, once the
|
||||
// indexer read it, its answer.
|
||||
// TakeInAsk is a value of the takeins state: which hand act a person asked, why, who and when, and, once the indexer read
|
||||
// it, its answer.
|
||||
type TakeInAsk struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module,omitempty"`
|
||||
// Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
|
||||
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
|
||||
CatchUp bool `json:"catch-up,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// SeatDefinitions asks a contracts module's trunk seat definitions to be read again now (indexer_seat_definitions_again,
|
||||
// DLV-I19).
|
||||
SeatDefinitions bool `json:"seat-definitions,omitempty"`
|
||||
@@ -91,51 +82,12 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil {
|
||||
if a.Module == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil {
|
||||
return a, errors.New("a takeins value naming no module, repository, pending wait or merge")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// NewTakeIn is the value indexer_take_in writes for a module: its key and the value.
|
||||
func NewTakeIn(module, why, by string, at time.Time) (string, []byte, error) {
|
||||
module = strings.TrimSpace(module)
|
||||
if module == "" {
|
||||
return "", nil, errors.New("name the module to take in (module)")
|
||||
}
|
||||
key, err := meshcatalogclient.ModulesKey(module)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Module: module, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// NewTakeInRepository is the value indexer_take_in writes for a repository: its key and the value.
|
||||
func NewTakeInRepository(repository, why, by string, at time.Time) (string, []byte, error) {
|
||||
repository = strings.TrimSpace(repository)
|
||||
key, err := trunk.RepositoryKey(repository)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "repository." + key, data, err
|
||||
}
|
||||
|
||||
// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first.
|
||||
const CatchUpKey = "catch-up"
|
||||
|
||||
// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every
|
||||
// repository the gate's source names (CatchUp).
|
||||
func NewCatchUp(why, by string, at time.Time) (string, []byte, error) {
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
why = "the catch-up after a restore"
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()})
|
||||
return CatchUpKey, data, err
|
||||
}
|
||||
|
||||
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
|
||||
const TakeInMergeTool = "indexer_take_in_merge"
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package index
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each
|
||||
// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them.
|
||||
type RepoRead struct {
|
||||
Root string
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
Packages map[string][]string
|
||||
}
|
||||
|
||||
// TakenIn is the answer of an ask of the state takeins: whether what was asked is done, of which module, from where, and
|
||||
// otherwise why not.
|
||||
type TakenIn struct {
|
||||
Module string `json:"module"`
|
||||
Stored bool `json:"stored"`
|
||||
Held bool `json:"held,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Answer string `json:"answer"`
|
||||
// Handed and Dropped are what a hand act on the catalog revisions moved and removed (revision_tools.go).
|
||||
Handed []string `json:"handed,omitempty"`
|
||||
Dropped []string `json:"dropped,omitempty"`
|
||||
// Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new.
|
||||
Fingerprints []FingerprintMove `json:"fingerprints,omitempty"`
|
||||
// Registry is the answer of a registrations' act (RegistryAct): the verb's own answer, as JSON.
|
||||
Registry json.RawMessage `json:"registry,omitempty"`
|
||||
}
|
||||
@@ -140,11 +140,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
|
||||
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the
|
||||
// catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves.
|
||||
// capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1. Design 47 H
|
||||
// §742 (H-A-105): no catch-up tool of the indexer's follows a restore, indexer_take_in having gone with the builder-log
|
||||
// index; the gate's gate_head_take is the one hand take of a head.
|
||||
if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 ||
|
||||
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) {
|
||||
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool)
|
||||
m.Objects[0].Form != 1 || m.Objects[0].CatchUp != "" {
|
||||
t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, no catch-up tool", m.Objects, index.CatalogRevisionsObjects)
|
||||
}
|
||||
// D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own
|
||||
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
|
||||
@@ -185,11 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
|
||||
// 1945: the hand acts are served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
|
||||
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules
|
||||
// kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read
|
||||
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
|
||||
want := []string{"indexer_take_in", "indexer_old_sections"}
|
||||
want := []string{"indexer_old_sections"}
|
||||
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
|
||||
want = append(want, v.Tool)
|
||||
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
|
||||
@@ -204,7 +205,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") {
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
@@ -224,7 +225,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("the indexer declares the state %s, which design 47 G retires", s.Name)
|
||||
}
|
||||
}
|
||||
for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads"} {
|
||||
for _, retired := range []string{"indexer_read_modules", "indexer_holds", "indexer_engine_reads", "indexer_take_in"} {
|
||||
if slices.Contains(m.Tools, retired) {
|
||||
t.Errorf("the indexer serves %s, which design 47 G retires", retired)
|
||||
}
|
||||
|
||||
@@ -27,7 +27,6 @@
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"indexer_take_in",
|
||||
"indexer_old_sections",
|
||||
"manifests",
|
||||
"modules",
|
||||
@@ -125,7 +124,7 @@
|
||||
{ "name": "mesh-delivery-indexer-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
|
||||
],
|
||||
"objects": [
|
||||
{ "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" }
|
||||
{ "name": "catalog-revisions", "max-mib": 512, "form": 1 }
|
||||
],
|
||||
"emits": [
|
||||
"take-in.pending",
|
||||
|
||||
Reference in New Issue
Block a user