diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index c79dd74b..c85abdb8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -2,23 +2,19 @@ package main import ( "context" + "encoding/json" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events -// take: mesh.mod..event.take-in., the wait's key in the body. A publish that fails is the caller's to say. -func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error { - return func(ctx context.Context, t takein.TakeIn) error { - event, err := takein.TakeInEvent(t.Kind) - if err != nil { - return err - } - data, err := takein.EncodeTakeIn(t) +// sayTakeIn says a take-in event of the seat mesh-catalog (design 47 H §743, §744) on the indexer's own subject, under the +// duplicate id given, so the same one said twice is heard once. +func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, string, string, any) error { + return func(ctx context.Context, event, id string, body any) error { + data, err := json.Marshal(body) if err != nil { return err } @@ -30,7 +26,7 @@ func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Contex msg.Data = data pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() - _, err = js.PublishMsg(pub, msg) + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) return err } } diff --git a/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go index 795f0850..889a565e 100644 --- a/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go @@ -29,7 +29,7 @@ func TestABuildOfAnotherWatchedBranchKeepsItsEdgesAndIsOutdatedAgainstThatBranch err := ix.KeepBranchEntries(ctx, "beta", source, map[string]meshcatalog.CatalogEntry{ "lib": {Module: "lib", Fp: libFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}}, "app": {Module: "app", Fp: appFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}}, - Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}}) + Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}}, nil) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index ee01742d..af2e80e7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) // ErrConflict is what a Store says when a write found the key changed or created since it was read. @@ -48,8 +47,6 @@ type Indexer struct { // BranchHead is the head of a repository's branch, asked of the seat git's verb branch: TakeIn reads a module a // registration names at its first watched branch's head (design 47 G A-85: the trunk is the first watched branch's). BranchHead func(ctx context.Context, repository, branch string) (string, error) - // keptView replaces KeepView for a test that counts the writes of the view; nil in the program. - keptView func(context.Context) error // Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296). Module string Store Store // candidates @@ -179,7 +176,6 @@ type Indexer struct { PinsPending func(ctx context.Context) (uint64, error) namesMu sync.Mutex // packageNames packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) - catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463) behindWaiting atomic.Bool // a source revision came while a catch-up ran (A-463a) behindAgain atomic.Bool // a source revision came while it ran @@ -195,8 +191,9 @@ type Indexer struct { // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. Heard Store judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) - // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. - SayTakeIn func(ctx context.Context, t takein.TakeIn) error + // SayTakeIn publishes a take-in event (take-in.taken, take-in.pending; design 47 H §743, §744) with its body, under the + // duplicate id given; nil says none. + SayTakeIn func(ctx context.Context, event, id string, body any) error // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store runMu sync.Mutex // one run event at a time (runcandidates.go) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 7a2325a5..5e239931 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -8,8 +8,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) -// Design 47 G A-85, §291: the indexer keeps a catalog revision per watched branch, trunk. for the first and -// branch.. (the branch escaped) for another, each numbered by its own branch, and the verb revision answers the +// Design 47 G A-85, §291, H §743, H-A-32: the indexer keeps a catalog revision per watched branch, . (the +// branch escaped) for every branch, the first included, numbered by the source revision it was taken in from, and the verb revision answers the // branch it is asked for; the event names the branch, escaped. func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { ctx := context.Background() @@ -22,13 +22,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { e.Commit = commit return map[string]meshcatalog.CatalogEntry{"reader": e} } - if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1")); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1"), nil); err != nil { t.Fatal(err) } - if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2")); err != nil { + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2"), nil); err != nil { t.Fatal(err) } - if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3")); err != nil { + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3"), nil); err != nil { t.Fatal(err) } names, _ := ix.Revisions.Names(ctx) @@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { for _, n := range names { have[n] = true } - for _, want := range []string{"trunk.1", "branch.feature/x=2ey.1", "branch.feature/x=2ey.2"} { + for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} { if !have[want] { t.Errorf("the objects hold %v, want %s", names, want) } } - if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 { - t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 { + t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision}) @@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { } return a } - if a := answer("main", 0); a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { + if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main answered %+v", a) } - if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { - t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision) } - if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's revision 1 answered %+v", a.Revision) + if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 7 answered %+v", a.Revision) } if a := answer("beta", 0); !a.NoneYet { t.Errorf("a branch with no revision answered %+v, want none-yet", a) @@ -68,13 +68,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source}) } - if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { + if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err) + if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } if _, err := bySource("feature/x=2ey", 9); err == nil { @@ -103,7 +103,7 @@ func TestTheRevisionAnswersEveryCaller(t *testing.T) { ctx := context.Background() ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20)} engine := meshcatalog.CatalogEntry{Module: "mesh-host", Repository: "novox/h", Commit: "c1", Manifest: map[string]any{}} - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}, nil); err != nil { t.Fatal(err) } if a, err := RevisionTool(ix.Revisions, nil).Run(map[string]any{"branch": "main"}); err != nil || a.(meshcatalog.RevisionAnswer).NoneYet { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go index 445ebad6..27eda37a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go @@ -43,7 +43,7 @@ func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) { refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest}) } b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs}) - if err := objects.Put(context.Background(), "trunk.1", b); err != nil { + if err := objects.Put(context.Background(), "main.1", b); err != nil { t.Fatal(err) } ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go index 1161e913..056a2df0 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go @@ -58,11 +58,11 @@ func TestAKeptRevisionHoldsTheEdgesOfItsEntries(t *testing.T) { err := ix.KeepBranchEntries(ctx, "main", 1, map[string]meshcatalog.CatalogEntry{ "b": {Module: "b", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "b:p"}}, Serves: []string{"seat:s.v"}}, "a": {Module: "a", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "a:p"}}, Invokes: []string{"seat:s.v"}}, - }) + }, nil) if err != nil { t.Fatal(err) } - _, entries, err := ReadRevision(ctx, ix.Revisions, "trunk.1") + _, entries, err := ReadRevision(ctx, ix.Revisions, "main.1") if err != nil { t.Fatal(err) } @@ -258,11 +258,11 @@ func TestAKeptRevisionHoldsTheActsAndTheKindFiveEdges(t *testing.T) { err := ix.KeepBranchEntries(ctx, "main", 1, map[string]meshcatalog.CatalogEntry{ "grants": {Module: "grants", Manifest: map[string]any{"module": "grants", "version": "1.0", "claims": []any{map[string]any{"name": "mesh-grants"}}}, Artifacts: []meshcatalog.Artifact{{Ref: "grants:p"}}}, "app": {Module: "app", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:p"}}}, - }) + }, nil) if err != nil { t.Fatal(err) } - _, entries, err := ReadRevision(ctx, ix.Revisions, "trunk.1") + _, entries, err := ReadRevision(ctx, ix.Revisions, "main.1") if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index cda30966..b76627a4 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,7 +22,7 @@ import ( ) // The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old -// `objects` section, its cap there): a revision is its list of entries' digests under trunk. or +// `objects` section, its cap there): a revision is its list of entries' digests under . or // candidate.., and each entry one object entry., written once and shared by every revision that // lists it. @@ -48,50 +49,46 @@ type Objects interface { // EntryObject is the object name of the entry of that digest. func EntryObject(digest string) string { return "entry." + digest } -// TrunkBranch is the first watched branch, whose revisions are kept as trunk. (design 47 G §291, A-85). +// TrunkBranch is the first watched branch (design 47 G §291, A-85). const TrunkBranch = "main" // isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named. func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch } -// BranchObject is the object name of a revision of a watched branch other than the first: branch.., -// the branch escaped as a name (A-85), so a dot in it never reads as a part. +// BranchObject is the object name of a revision of a watched branch, the trunk included: ., the branch +// escaped as a name, so a dot in it never reads as a part, and the revision the source revision it was taken in from (design +// 47 H §743, H-A-32). func BranchObject(branch string, revision int64) string { - return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) + if branch == "" { + branch = TrunkBranch + } + return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } -// RevisionObject is the object name of a revision: trunk. for the first watched branch, -// branch.. for another watched branch (A-85), or candidate.. for a candidate, -// which keeps its newest revision only. +// RevisionObject is the object name of a revision: . for a watched branch (BranchObject), or +// candidate.. for a candidate, which keeps its newest revision only. func RevisionObject(rev meshcatalog.CatalogRevision) string { if rev.Kind == "candidate" { return "candidate." + rev.Group + "." + rev.Heads } - if !isTrunkBranch(rev.Branch) { - return BranchObject(rev.Branch, rev.Revision) - } - return "trunk." + strconv.FormatInt(rev.Revision, 10) + return BranchObject(rev.Branch, rev.Revision) } // BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one. func BranchRevisionOf(branch, name string) (int64, bool) { - n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".") + if branch == "" { + branch = TrunkBranch + } + n, ok := strings.CutPrefix(name, keys.EscapeName(branch)+".") if !ok { return 0, false } r, err := strconv.ParseInt(n, 10, 64) - return r, err == nil + return r, err == nil && r > 0 } // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. -func TrunkRevisionOf(name string) (int64, bool) { - n, ok := strings.CutPrefix(name, "trunk.") - if !ok { - return 0, false - } - r, err := strconv.ParseInt(n, 10, 64) - return r, err == nil -} +func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) } // KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the // revision's list, so a list is never kept whose entries are not; an entry already held is not written again. @@ -166,10 +163,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo if err != nil { return meshcatalog.CatalogRevision{}, false, err } - number := TrunkRevisionOf - if !isTrunkBranch(branch) { - number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } - } + number := func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } var newest int64 for _, n := range names { if r, ok := number(n); ok && r > newest { @@ -236,49 +230,6 @@ func changedEntries(before, after []meshcatalog.EntryRef) []string { return changed } -// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two -// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the -// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not -// kept is said in the log and returned, so the take-in that made it is not said taken. -func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error { - if ix.Revisions == nil { - return nil - } - // Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions - // and inputs written), so the revision holds what the next one is computed from. - held, err := ix.heldModules(ctx) - if err != nil { - return err - } - // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. - entries := make(map[string]meshcatalog.CatalogEntry, len(held)) - for name, cur := range held { - entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]), edges[name].manifest) - } - for name, e := range edges { - if _, ok := held[name]; !ok { - entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]), e.manifest) - } - } - return ix.keepEntries(ctx, entries) -} - -// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on -// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer -// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of -// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest -// revision's keeps none. -func (ix *Indexer) KeepView(ctx context.Context) error { - if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() { - return nil - } - entries, err := ix.viewEntries(ctx) - if err != nil { - return err - } - return ix.keepEntries(ctx, entries) -} - // viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value // with its node of the graph, as KeepView keeps them. func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) { @@ -314,50 +265,47 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata return entries, nil } -// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry -// moved; one keep at a time, so two never take the same number. -func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error { - if ix.catchingUp.Load() { - return nil // the catch-up of a first start keeps its one revision at its end (CatchUp) +// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design +// 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the +// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, +// pipeline.., in the same write when there is one (A-449a). A revision already kept under that name is not written +// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. +func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { + return ix.keepBranchEntries(ctx, branch, src, entries, movers) +} + +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { + if ix.Revisions == nil { + return nil } - ix.mu.Lock() - src := int64(ix.source.rev) - ix.mu.Unlock() - return ix.keepBranchEntries(ctx, TrunkBranch, src, entries) -} - -// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291, -// A-85): trunk. for the first watched branch, branch.. for another, numbered per branch, and -// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from. -func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { - return ix.keepBranchEntries(ctx, branch, src, entries) -} - -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { if branch == "" { branch = TrunkBranch } ix.keepMu.Lock() defer ix.keepMu.Unlock() - newest, any, err := newestOf(ctx, ix.Revisions, branch) + newest, _, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) - return nil + return err } if err != nil { ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) return fmt.Errorf("the newest catalog graph revision: %w", err) } + if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil { + ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src)) + return nil + } else if !errors.Is(err, ErrNoObject) { + return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err) + } entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) if err != nil { return err } + rev.Movers = movers changed := changedEntries(newest.Entries, rev.Entries) - if (any && len(changed) == 0) || (!any && len(entries) == 0) { - return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs - } used, capacity, fillErr := ix.Revisions.Fill(ctx) if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { // W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the @@ -375,6 +323,14 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) return err } + // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). + if len(movers) > 0 && ix.Pins != nil { + record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) + if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { + ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) + return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) + } + } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. if removed, err := ix.judgePinsLocked(ctx); err != nil { ix.logf("pins: the revisions no record pins are not removed: %v", err) diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go index a8925218..5de12ff7 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go @@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) { ctx := context.Background() fp := "src2:" + strings.Repeat("d", 64) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" { diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index d691af6b..bf8bb1cf 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -28,11 +28,8 @@ import ( // Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that // resolves them writes the import form, one rebuild too many, never a stale module. func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { - wrote, err := ix.applyBuildEnd(ctx, value) - if err != nil || !wrote { - return err // an entry that wrote nothing leaves the view as it is - } - return ix.keepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back + _, err := ix.applyBuildEnd(ctx, value) + return err // the imports reach the next take-in's revision (design 47 H §743, H-A-49) } // ApplyBuildEndsAtStart applies every entry of the build log read at start as ApplyBuildEnd does, and keeps the view @@ -63,20 +60,8 @@ func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, f wrote++ } } - var err error - if wrote > 0 { - err = ix.keepView(ctx) - } ix.logf("replayed %d build ends at start: %d wrote a module, in %s", len(values), wrote, time.Since(began).Round(time.Millisecond)) - return err -} - -// keepView is KeepView; a test replaces it to count the writes of the view. -func (ix *Indexer) keepView(ctx context.Context) error { - if ix.keptView != nil { - return ix.keptView(ctx) - } - return ix.KeepView(ctx) + return nil } // applyBuildEnd reports whether it wrote the module (its final write changed the held value). diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 456c2f48..4f308265 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -401,43 +401,3 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * } }) } - -// A start replays every build end of the build log: an entry that writes no module keeps no view, and the entries that -// write keep it once, after all of them (measured: the view kept per entry cost 50 min CPU per start). -func TestBuildEndsKeepTheViewOnlyWhenTheyWrite(t *testing.T) { - ctx := context.Background() - ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - if err := ix.Recompute(ctx); err != nil { - t.Fatal(err) - } - raw, _, _, _ := ix.Modules.Get(ctx, "avahi") - cur, err := meshcatalogclient.DecodeModule(raw) - if err != nil || cur.Fp2 == "" { - t.Fatalf("no fingerprint to start from: %v %q", err, cur.None2) - } - older := buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK) - writes := buildEnd(t, "avahi", cur.Fp2, meshSDK) - kept := 0 - ix.keptView = func(context.Context) error { kept++; return nil } - for _, c := range []struct { - name string - entries [][]byte - want int - }{ - {"no entry", nil, 0}, - {"entries that write nothing", [][]byte{older, []byte("not a build entry"), older}, 0}, - {"a write among entries that write nothing", [][]byte{older, writes, older}, 1}, - {"the same write again keeps nothing", [][]byte{writes, older}, 0}, - } { - kept = 0 - if err := ix.ApplyBuildEndsAtStart(ctx, c.entries, nil); err != nil || kept != c.want { - t.Errorf("%s: the view was kept %d times (%v), want %d", c.name, kept, err, c.want) - } - } - // The live path, one entry: the view is kept when it wrote. - kept = 0 - if err := ix.ApplyBuildEnd(ctx, older); err != nil || kept != 0 { - t.Errorf("a live entry that writes nothing kept the view %d times (%v)", kept, err) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index f871def7..3b585be1 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -270,7 +270,7 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t tool = got.(map[string]any) } } - return tool != nil && tool["name"] == "trunk.2", nil + return tool != nil && tool["name"] == "main.2", nil }}, "Outdated": {"", func(ix *Indexer) (bool, error) { a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{}) diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index ee7ece61..42f9e91e 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -279,10 +279,7 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ // message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is // and said: a failure never empties the bucket. func (ix *Indexer) Reconcile(ctx context.Context) error { - if err := ix.reconcile(ctx); err != nil { - return err - } - return ix.KeepView(ctx) // the modules dropped leave the view's next revision too + return ix.reconcile(ctx) // the modules dropped leave the next take-in's revision (design 47 H §743, H-A-49) } func (ix *Indexer) reconcile(ctx context.Context) error { diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 2fce27e8..151cfbae 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -180,18 +180,18 @@ func (ix *Indexer) clearPinMissing(ctx context.Context, key string) { // RevisionGroup says which watched branch's line a revision object belongs to, and its number; false for a candidate or an entry. func RevisionGroup(name string) (group string, revision int64, ok bool) { - if n, ok := TrunkRevisionOf(name); ok { - return "", n, true + if strings.HasPrefix(name, "candidate.") || strings.HasPrefix(name, "entry.") { + return "", 0, false } - if rest, ok := strings.CutPrefix(name, "branch."); ok { - i := strings.LastIndex(rest, ".") - if i > 0 { - if n, err := strconv.ParseInt(rest[i+1:], 10, 64); err == nil { - return rest[:i], n, true - } - } + i := strings.LastIndex(name, ".") + if i <= 0 { + return "", 0, false } - return "", 0, false + n, err := strconv.ParseInt(name[i+1:], 10, 64) + if err != nil || n <= 0 { + return "", 0, false + } + return name[:i], n, true } // CurrentRevisions are the kept revision objects that are current (§579): the newest of each watched branch, and every open @@ -474,22 +474,3 @@ func ConsumerTools(st Store) []stdio.Tool { Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}} } - -// PinPipelines pins the catalog graph revision a take-in made for the one pipeline the take-in starts, under the record -// `pipeline..` (the runner's key, design 47 H §743, A-449a), in the indexer's own pins state: so no later -// take-in removes the revision before the runner starts it. The pipeline's pipeline-started names the same record (a set), -// and its pipeline-ended removes it (§749). It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a -// source, an import) can make it non-current and remove it between the two; it answers that revision, 0 when none is kept. -func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64, entries []meshcatalog.TakeInEntry) int64 { - ix.keepMu.Lock() - defer ix.keepMu.Unlock() - revision := ix.NewestRevision(ctx, branch) - if ix.Pins == nil || revision <= 0 || len(entries) == 0 { - return revision - } - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))) - if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil { - ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err) - } - return revision -} diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index fb3172d6..d0ae03f6 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -44,35 +44,35 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } } reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} - keep := func(commit string) { + keep := func(commit string, source int64) { t.Helper() e := reader e.Commit = commit - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil { t.Fatal(err) } } - keep("c1") + keep("c1", 1) pending = 3 // pinned before the next keeps, which judge the revisions while the consumers are behind started("main.5", 1) started("main.5", 1) // heard twice: pins once started("main.6", 1) - keep("c2") - keep("c3") + keep("c2", 2) + keep("c3", 3) ix.JudgePins(ctx) - want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3") + want("while messages of the consumer are pending", "main.1", "main.2", "main.3") pending = 0 ix.JudgePins(ctx) - want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3") + want("pinned by two records, the middle one neither pinned nor current", "main.1", "main.3") if v, _, _, _ := ix.Pins.Get(ctx, "main.1"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5", "pipeline.main.6"}) { t.Errorf("the pin of main.1 holds %s, want the two pipelines' records once each", v) } ended("main.5", 1) ix.JudgePins(ctx) - want("one of two records freed it", "trunk.1", "trunk.3") + want("one of two records freed it", "main.1", "main.3") ended("main.6", 1) ix.JudgePins(ctx) - want("both freed it", "trunk.3") + want("both freed it", "main.3") if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v, want no key of the removed revision", keys) } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index a21bbd3c..419faf50 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -7,7 +7,6 @@ import ( "fmt" "regexp" "slices" - "strconv" "strings" "time" @@ -330,7 +329,7 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA case str(args, "group") != "" || str(args, "heads") != "": name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) case n > 0: - name = "trunk." + strconv.FormatInt(int64(n), 10) + name = BranchObject(TrunkBranch, int64(n)) default: newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go new file mode 100644 index 00000000..115b121c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -0,0 +1,117 @@ +package index + +import ( + "context" + "errors" + "fmt" + "maps" + "strconv" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrOldForm is a source-moved of the earlier form, one with no field its form requires (design 47 H §765, H-A-58): it is +// refused, said in the journal, and never taken in. +var ErrOldForm = errors.New("a source-moved of the old form") + +// TakenID is the duplicate id take-in.taken of a branch's source revision is said under, so a take-in said again is heard once. +func TakenID(branch string, source int64) string { + return takein.TakeInTakenEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) +} + +// TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each +// repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules +// registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch, +// ., carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and +// take-in.taken once, with branch, source, revision, call and one entry per mover. A revision already kept under that name (a +// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched +// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. +// An error leaves the source-moved held: ErrOldForm never takes it in, ErrObjectsFull waits for room (§744), any other is the +// failed read or write it says. +func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) { + taken := meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call} + if e.Branch == "" || e.Source <= 0 || e.Call == "" { + return taken, fmt.Errorf("%w: branch %q, source %d, call %q", ErrOldForm, e.Branch, e.Source, e.Call) + } + if ix.Revisions == nil { + return taken, errors.New("the indexer keeps no catalog revisions") + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + name := BranchObject(e.Branch, e.Source) + if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil { + taken.Revision, taken.Entries = kept.Revision, kept.Movers + ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers)) + return taken, ix.sayTaken(ctx, taken) + } else if !errors.Is(err, ErrNoObject) { + return taken, err + } + _, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch) + if err != nil { + return taken, fmt.Errorf("the newest catalog revision of %s: %w", e.Branch, err) + } + x := ix + if !isTrunkBranch(e.Branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, e.Branch) + if err != nil { + return taken, err + } + d := ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("take-in of "+e.Branch+": "+format, args...) } + x = d.ix + } + heads := map[string]string{} + var moved []string + for _, h := range e.Heads { + heads[h.Repository], moved = h.Commit, append(moved, h.Repository) + } + x.mu.Lock() + x.source = sourceNow{rev: uint64(e.Source), heads: mergedHeads(x.source.heads, heads)} + x.mu.Unlock() + for _, h := range e.Heads { + f, err := x.wholeAt(ctx, h.Repository, h.Commit) + if err != nil { + return taken, err + } + f.Branch = e.Branch + if _, err := x.ApplyMessage(ctx, f); err != nil { + return taken, fmt.Errorf("taking %s at %.12s in: %w", h.Repository, h.Commit, err) + } + } + entries, err := x.viewEntries(ctx) + if err != nil { + return taken, err + } + movers := moversOf(before, entries, moved) + if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil { + return taken, err + } + taken.Revision, taken.Entries = e.Source, movers + ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers)) + return taken, ix.sayTaken(ctx, taken) +} + +// mergedHeads is held with the heads given laid over it. +func mergedHeads(held, over map[string]string) map[string]string { + out := maps.Clone(held) + if out == nil { + out = map[string]string{} + } + maps.Copy(out, over) + return out +} + +// sayTaken says take-in.taken under its duplicate id; nil SayTakeIn says nothing. +func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) error { + if ix.SayTakeIn == nil { + return nil + } + if err := ix.SayTakeIn(ctx, takein.TakeInTakenEvent, TakenID(t.Branch, t.Source), t); err != nil { + return fmt.Errorf("take-in.taken of %s: %w", BranchObject(t.Branch, t.Source), err) + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go new file mode 100644 index 00000000..6d0d4e24 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -0,0 +1,94 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" +) + +// takeInIndexer is an indexer that takes source revisions in from the catalogue repository's trees at f's commit, keeping its +// revisions and pins, and the take-in events it says. +func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTaken, whole) { + t.Helper() + ix, _, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + objects := newMemObjects(1 << 22) + ix.Revisions, ix.Pins, ix.PinsPending = objects, newMem(), func(context.Context) (uint64, error) { return 0, nil } + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + if repository != f.Repository || sha != f.Merge { + return RepoRead{}, errors.New("no such commit") + } + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil } + said := &[]meshcatalog.TakeInTaken{} + ix.SayTakeIn = func(_ context.Context, event, id string, body any) error { + tk, ok := body.(meshcatalog.TakeInTaken) + if event != "take-in.taken" || !ok || id != TakenID(tk.Branch, tk.Source) { + t.Errorf("said %s %s %+v, want take-in.taken under its branch and source", event, id, body) + } + *said = append(*said, tk) + return nil + } + return ix, objects, said, f +} + +// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision . carrying its movers, pins it for the +// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover +// pins nothing and says no entries; the old form is refused. +func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { + ctx := context.Background() + ix, objects, said, f := takeInIndexer(t) + moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 7, Call: "0123456789abcdef", + Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}} + taken, err := ix.TakeInSource(ctx, moved) + if err != nil { + t.Fatal(err) + } + if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge || + taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { + t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken) + } + rev, _, err := ReadRevision(ctx, objects, "main.7") + if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { + t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err) + } + if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found { + t.Error("main.7 is not pinned for its pipeline") + } else { + var pv PinValue + if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) { + t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v) + } + } + puts := objects.puts + again, err := ix.TakeInSource(ctx, moved) + if err != nil || objects.puts != puts || !reflect.DeepEqual(again, taken) { + t.Errorf("a redelivery wrote %d objects and said %+v (%v), want none and the same", objects.puts-puts, again, err) + } + same := moved + same.Source, same.Call = 8, "fedcba9876543210" + none, err := ix.TakeInSource(ctx, same) + if err != nil || len(none.Entries) != 0 || none.Revision != 8 { + t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err) + } + if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found { + t.Error("a take-in with no mover pinned its revision") + } + if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil { + t.Errorf("main.8 is not kept: %v", err) + } + if len(*said) != 3 { + t.Errorf("said %d take-in.taken, want 3", len(*said)) + } + old := moved + old.Call = "" + if _, err := ix.TakeInSource(ctx, old); !errors.Is(err, ErrOldForm) { + t.Errorf("a source-moved with no call answered %v, want ErrOldForm", err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 40019e0f..4ebd5ce8 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -372,7 +372,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc if err := ix.writeGraph(ctx, edges); err != nil { return err } - return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges, ix.BuildModules())) + return nil // only a take-in keeps a catalog revision (design 47 H §743, H-A-49) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 83f6b032..6db6ed10 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -384,9 +384,9 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) { t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts) } - if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || + if back, got, err := ReadRevision(context.Background(), kept, "main.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || !slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) { - t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err) + t.Errorf("main.2 read back = %+v, %d entries, %v", back, len(got), err) } // W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its // entry gives it back; a bucket that keeps none is an empty view at 0. @@ -414,36 +414,31 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err) } _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) - if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { - t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) + if _, _, err := ReadRevision(context.Background(), kept, "main.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { + t.Errorf("main.2 with a lost entry read without error naming it: %v", err) } - // W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps - // none; a module leaving makes the next revision, naming it changed. + // Design 47 H §743, H-A-49: only a take-in keeps a catalog revision; a recompute keeps none and says no catalog-revision. ix.Revisions = newMemObjects(0) var revisions []meshcatalog.CatalogRevisionEvent ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { revisions = append(revisions, e) return nil } - if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" || - revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) { - t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err) + if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 0 { + t.Fatalf("a recompute said %+v (%v), want no revision", revisions, err) } graph := ix.Graph.(*memStore) written := graph.writes - if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 { - t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err) + if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { + t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err) } // A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute. delete(mods.vals, "other") if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } - if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) { - t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions) - } - if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) { - t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals)) + if names, _ := ix.Revisions.Names(context.Background()); len(names) != 0 { + t.Errorf("recomputes kept %v, want nothing", names) } if _, held := graph.vals["other"]; held { t.Error("graph.other is still held after its module went") @@ -506,11 +501,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if err := first.Apply(context.Background(), merged); err != nil { t.Fatal(err) } - if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || + takeIn := func(source int64) error { + entries, err := first.viewEntries(context.Background()) + if err != nil { + return err + } + return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil) + } + if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 || firstSaid[0].Entries != int64(len(builderModules(t))) { t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } - // KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision. + // A write to the view between take-ins (here a verdict carried into one module) is in the next take-in's revision. viewKey, _ := meshcatalogclient.ModulesKey("avahi") _ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) { m, _ := meshcatalogclient.DecodeModule(value) @@ -518,11 +520,11 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // data, err := meshcatalogclient.EncodeModule(m) return data, false, err }) - if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { - t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err) + if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { + t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err) } - if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { - t.Errorf("the view at a restart is read at %d (%v), want 2", at, err) + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 { + t.Errorf("the view at a restart is read at %d (%v), want 7", at, err) } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } @@ -538,21 +540,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("the first-view tools %+v serve no modules", FirstViewTools) } if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 2 || len(a.Values) != len(builderModules(t)) { - t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err) + a.Revision != 7 || len(a.Values) != len(builderModules(t)) { + t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err) } else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" { t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err) } - if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { - t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err) } // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { - t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) + a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err) } if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index f3d23fc8..cdc1fc3f 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -102,10 +102,10 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( n := NewNamed(newMem(), nil) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} n.Changed = ix.RemoveUnlisted - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } - if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil { // the revision that listed it is removed + if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed t.Fatal(err) } found := func() bool { @@ -213,10 +213,10 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin fp := namedFp("c") setup := func(n *Named) (*Indexer, func() bool) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } - if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil { + if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { t.Fatal(err) } return ix, func() bool {