From abdd70d9180f8790179bfeca8b1ecf6c44fa0cfc Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 03:43:38 +0100 Subject: [PATCH] I2b: one catalog revision per take-in, ., carrying its movers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TakeInSource takes a gate's source-moved in: the trees of each repository it moves at the named commit through the seat git, the fingerprints of the modules registered for it, the movers against the branch's newest catalog revision, and one revision . for every branch, the trunk included, numbered by the source and written once by read before write. The revision carries its movers, and a redelivery whose revision is kept says take-in.taken again from them, recomputing nothing (H-A-92). It is pinned for the take-in's pipeline as pipeline.. in the same write when it has a mover, none for a take-in with no mover. take-in.taken carries branch, source, revision, call and the movers' entries alone, under a duplicate id of its branch and source (H-A-31). A source-moved with no call is refused as the old form (§765). Only a take-in keeps a catalog revision (H-A-49): the recompute, the build ends' imports and the start's reconcile keep none, and KeepView, keepEntries, keepTrunkRevision and PinPipelines go. Another branch than the first is taken in on its newest revision's view. Builds: design 47 H §743, §749 (the pin in the take-in's write), §765 (the old form); H-A-31, H-A-32, H-A-49, H-A-92 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer/verdicts.go | 18 +-- .../internal/index/branch_builds_test.go | 2 +- .../internal/index/candidates.go | 9 +- .../internal/index/catalog_branches_test.go | 36 ++--- .../internal/index/catalog_calls_test.go | 2 +- .../internal/index/catalog_edges_test.go | 8 +- .../internal/index/catalog_revisions.go | 144 ++++++------------ .../internal/index/entry_verb_test.go | 2 +- .../internal/index/imports.go | 21 +-- .../internal/index/imports_test.go | 40 ----- .../internal/index/kept_trunk_test.go | 2 +- .../internal/index/modules.go | 5 +- .../internal/index/pins.go | 39 ++--- .../internal/index/pins_test.go | 18 +-- .../internal/index/revision_tools.go | 3 +- .../internal/index/takein_source.go | 117 ++++++++++++++ .../internal/index/takein_source_test.go | 94 ++++++++++++ .../internal/index/toolchains.go | 2 +- .../internal/index/toolchains_test.go | 58 +++---- .../internal/index/wanted_names_test.go | 8 +- 20 files changed, 356 insertions(+), 272 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/takein_source.go create mode 100644 modules/mesh-delivery-indexer/internal/index/takein_source_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index c79dd74b..c85abdb8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -2,23 +2,19 @@ package main import ( "context" + "encoding/json" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events -// take: mesh.mod..event.take-in., the wait's key in the body. A publish that fails is the caller's to say. -func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error { - return func(ctx context.Context, t takein.TakeIn) error { - event, err := takein.TakeInEvent(t.Kind) - if err != nil { - return err - } - data, err := takein.EncodeTakeIn(t) +// sayTakeIn says a take-in event of the seat mesh-catalog (design 47 H §743, §744) on the indexer's own subject, under the +// duplicate id given, so the same one said twice is heard once. +func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, string, string, any) error { + return func(ctx context.Context, event, id string, body any) error { + data, err := json.Marshal(body) if err != nil { return err } @@ -30,7 +26,7 @@ func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Contex msg.Data = data pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() - _, err = js.PublishMsg(pub, msg) + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) return err } } diff --git a/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go index 795f0850..889a565e 100644 --- a/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go @@ -29,7 +29,7 @@ func TestABuildOfAnotherWatchedBranchKeepsItsEdgesAndIsOutdatedAgainstThatBranch err := ix.KeepBranchEntries(ctx, "beta", source, map[string]meshcatalog.CatalogEntry{ "lib": {Module: "lib", Fp: libFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}}, "app": {Module: "app", Fp: appFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}}, - Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}}) + Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}}, nil) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index ee01742d..af2e80e7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) // ErrConflict is what a Store says when a write found the key changed or created since it was read. @@ -48,8 +47,6 @@ type Indexer struct { // BranchHead is the head of a repository's branch, asked of the seat git's verb branch: TakeIn reads a module a // registration names at its first watched branch's head (design 47 G A-85: the trunk is the first watched branch's). BranchHead func(ctx context.Context, repository, branch string) (string, error) - // keptView replaces KeepView for a test that counts the writes of the view; nil in the program. - keptView func(context.Context) error // Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296). Module string Store Store // candidates @@ -179,7 +176,6 @@ type Indexer struct { PinsPending func(ctx context.Context) (uint64, error) namesMu sync.Mutex // packageNames packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) - catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463) behindWaiting atomic.Bool // a source revision came while a catch-up ran (A-463a) behindAgain atomic.Bool // a source revision came while it ran @@ -195,8 +191,9 @@ type Indexer struct { // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. Heard Store judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) - // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. - SayTakeIn func(ctx context.Context, t takein.TakeIn) error + // SayTakeIn publishes a take-in event (take-in.taken, take-in.pending; design 47 H §743, §744) with its body, under the + // duplicate id given; nil says none. + SayTakeIn func(ctx context.Context, event, id string, body any) error // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store runMu sync.Mutex // one run event at a time (runcandidates.go) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 7a2325a5..5e239931 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -8,8 +8,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) -// Design 47 G A-85, §291: the indexer keeps a catalog revision per watched branch, trunk. for the first and -// branch.. (the branch escaped) for another, each numbered by its own branch, and the verb revision answers the +// Design 47 G A-85, §291, H §743, H-A-32: the indexer keeps a catalog revision per watched branch, . (the +// branch escaped) for every branch, the first included, numbered by the source revision it was taken in from, and the verb revision answers the // branch it is asked for; the event names the branch, escaped. func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { ctx := context.Background() @@ -22,13 +22,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { e.Commit = commit return map[string]meshcatalog.CatalogEntry{"reader": e} } - if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1")); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1"), nil); err != nil { t.Fatal(err) } - if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2")); err != nil { + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2"), nil); err != nil { t.Fatal(err) } - if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3")); err != nil { + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3"), nil); err != nil { t.Fatal(err) } names, _ := ix.Revisions.Names(ctx) @@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { for _, n := range names { have[n] = true } - for _, want := range []string{"trunk.1", "branch.feature/x=2ey.1", "branch.feature/x=2ey.2"} { + for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} { if !have[want] { t.Errorf("the objects hold %v, want %s", names, want) } } - if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 { - t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 { + t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision}) @@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { } return a } - if a := answer("main", 0); a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { + if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main answered %+v", a) } - if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { - t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision) } - if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's revision 1 answered %+v", a.Revision) + if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 7 answered %+v", a.Revision) } if a := answer("beta", 0); !a.NoneYet { t.Errorf("a branch with no revision answered %+v, want none-yet", a) @@ -68,13 +68,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source}) } - if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { + if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err) + if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } if _, err := bySource("feature/x=2ey", 9); err == nil { @@ -103,7 +103,7 @@ func TestTheRevisionAnswersEveryCaller(t *testing.T) { ctx := context.Background() ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20)} engine := meshcatalog.CatalogEntry{Module: "mesh-host", Repository: "novox/h", Commit: "c1", Manifest: map[string]any{}} - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}, nil); err != nil { t.Fatal(err) } if a, err := RevisionTool(ix.Revisions, nil).Run(map[string]any{"branch": "main"}); err != nil || a.(meshcatalog.RevisionAnswer).NoneYet { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go index 445ebad6..27eda37a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go @@ -43,7 +43,7 @@ func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) { refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest}) } b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs}) - if err := objects.Put(context.Background(), "trunk.1", b); err != nil { + if err := objects.Put(context.Background(), "main.1", b); err != nil { t.Fatal(err) } ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go index 1161e913..056a2df0 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_edges_test.go @@ -58,11 +58,11 @@ func TestAKeptRevisionHoldsTheEdgesOfItsEntries(t *testing.T) { err := ix.KeepBranchEntries(ctx, "main", 1, map[string]meshcatalog.CatalogEntry{ "b": {Module: "b", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "b:p"}}, Serves: []string{"seat:s.v"}}, "a": {Module: "a", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "a:p"}}, Invokes: []string{"seat:s.v"}}, - }) + }, nil) if err != nil { t.Fatal(err) } - _, entries, err := ReadRevision(ctx, ix.Revisions, "trunk.1") + _, entries, err := ReadRevision(ctx, ix.Revisions, "main.1") if err != nil { t.Fatal(err) } @@ -258,11 +258,11 @@ func TestAKeptRevisionHoldsTheActsAndTheKindFiveEdges(t *testing.T) { err := ix.KeepBranchEntries(ctx, "main", 1, map[string]meshcatalog.CatalogEntry{ "grants": {Module: "grants", Manifest: map[string]any{"module": "grants", "version": "1.0", "claims": []any{map[string]any{"name": "mesh-grants"}}}, Artifacts: []meshcatalog.Artifact{{Ref: "grants:p"}}}, "app": {Module: "app", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:p"}}}, - }) + }, nil) if err != nil { t.Fatal(err) } - _, entries, err := ReadRevision(ctx, ix.Revisions, "trunk.1") + _, entries, err := ReadRevision(ctx, ix.Revisions, "main.1") if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index cda30966..b76627a4 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,7 +22,7 @@ import ( ) // The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old -// `objects` section, its cap there): a revision is its list of entries' digests under trunk. or +// `objects` section, its cap there): a revision is its list of entries' digests under . or // candidate.., and each entry one object entry., written once and shared by every revision that // lists it. @@ -48,50 +49,46 @@ type Objects interface { // EntryObject is the object name of the entry of that digest. func EntryObject(digest string) string { return "entry." + digest } -// TrunkBranch is the first watched branch, whose revisions are kept as trunk. (design 47 G §291, A-85). +// TrunkBranch is the first watched branch (design 47 G §291, A-85). const TrunkBranch = "main" // isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named. func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch } -// BranchObject is the object name of a revision of a watched branch other than the first: branch.., -// the branch escaped as a name (A-85), so a dot in it never reads as a part. +// BranchObject is the object name of a revision of a watched branch, the trunk included: ., the branch +// escaped as a name, so a dot in it never reads as a part, and the revision the source revision it was taken in from (design +// 47 H §743, H-A-32). func BranchObject(branch string, revision int64) string { - return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) + if branch == "" { + branch = TrunkBranch + } + return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } -// RevisionObject is the object name of a revision: trunk. for the first watched branch, -// branch.. for another watched branch (A-85), or candidate.. for a candidate, -// which keeps its newest revision only. +// RevisionObject is the object name of a revision: . for a watched branch (BranchObject), or +// candidate.. for a candidate, which keeps its newest revision only. func RevisionObject(rev meshcatalog.CatalogRevision) string { if rev.Kind == "candidate" { return "candidate." + rev.Group + "." + rev.Heads } - if !isTrunkBranch(rev.Branch) { - return BranchObject(rev.Branch, rev.Revision) - } - return "trunk." + strconv.FormatInt(rev.Revision, 10) + return BranchObject(rev.Branch, rev.Revision) } // BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one. func BranchRevisionOf(branch, name string) (int64, bool) { - n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".") + if branch == "" { + branch = TrunkBranch + } + n, ok := strings.CutPrefix(name, keys.EscapeName(branch)+".") if !ok { return 0, false } r, err := strconv.ParseInt(n, 10, 64) - return r, err == nil + return r, err == nil && r > 0 } // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. -func TrunkRevisionOf(name string) (int64, bool) { - n, ok := strings.CutPrefix(name, "trunk.") - if !ok { - return 0, false - } - r, err := strconv.ParseInt(n, 10, 64) - return r, err == nil -} +func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) } // KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the // revision's list, so a list is never kept whose entries are not; an entry already held is not written again. @@ -166,10 +163,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo if err != nil { return meshcatalog.CatalogRevision{}, false, err } - number := TrunkRevisionOf - if !isTrunkBranch(branch) { - number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } - } + number := func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } var newest int64 for _, n := range names { if r, ok := number(n); ok && r > newest { @@ -236,49 +230,6 @@ func changedEntries(before, after []meshcatalog.EntryRef) []string { return changed } -// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two -// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the -// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not -// kept is said in the log and returned, so the take-in that made it is not said taken. -func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error { - if ix.Revisions == nil { - return nil - } - // Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions - // and inputs written), so the revision holds what the next one is computed from. - held, err := ix.heldModules(ctx) - if err != nil { - return err - } - // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. - entries := make(map[string]meshcatalog.CatalogEntry, len(held)) - for name, cur := range held { - entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]), edges[name].manifest) - } - for name, e := range edges { - if _, ok := held[name]; !ok { - entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]), e.manifest) - } - } - return ix.keepEntries(ctx, entries) -} - -// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on -// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer -// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of -// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest -// revision's keeps none. -func (ix *Indexer) KeepView(ctx context.Context) error { - if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() { - return nil - } - entries, err := ix.viewEntries(ctx) - if err != nil { - return err - } - return ix.keepEntries(ctx, entries) -} - // viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value // with its node of the graph, as KeepView keeps them. func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) { @@ -314,50 +265,47 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata return entries, nil } -// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry -// moved; one keep at a time, so two never take the same number. -func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error { - if ix.catchingUp.Load() { - return nil // the catch-up of a first start keeps its one revision at its end (CatchUp) +// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design +// 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the +// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, +// pipeline.., in the same write when there is one (A-449a). A revision already kept under that name is not written +// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. +func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { + return ix.keepBranchEntries(ctx, branch, src, entries, movers) +} + +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { + if ix.Revisions == nil { + return nil } - ix.mu.Lock() - src := int64(ix.source.rev) - ix.mu.Unlock() - return ix.keepBranchEntries(ctx, TrunkBranch, src, entries) -} - -// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291, -// A-85): trunk. for the first watched branch, branch.. for another, numbered per branch, and -// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from. -func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { - return ix.keepBranchEntries(ctx, branch, src, entries) -} - -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { if branch == "" { branch = TrunkBranch } ix.keepMu.Lock() defer ix.keepMu.Unlock() - newest, any, err := newestOf(ctx, ix.Revisions, branch) + newest, _, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) - return nil + return err } if err != nil { ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) return fmt.Errorf("the newest catalog graph revision: %w", err) } + if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil { + ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src)) + return nil + } else if !errors.Is(err, ErrNoObject) { + return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err) + } entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) if err != nil { return err } + rev.Movers = movers changed := changedEntries(newest.Entries, rev.Entries) - if (any && len(changed) == 0) || (!any && len(entries) == 0) { - return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs - } used, capacity, fillErr := ix.Revisions.Fill(ctx) if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { // W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the @@ -375,6 +323,14 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) return err } + // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). + if len(movers) > 0 && ix.Pins != nil { + record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) + if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { + ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) + return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) + } + } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. if removed, err := ix.judgePinsLocked(ctx); err != nil { ix.logf("pins: the revisions no record pins are not removed: %v", err) diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go index a8925218..5de12ff7 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go @@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) { ctx := context.Background() fp := "src2:" + strings.Repeat("d", 64) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" { diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index d691af6b..bf8bb1cf 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -28,11 +28,8 @@ import ( // Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that // resolves them writes the import form, one rebuild too many, never a stale module. func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { - wrote, err := ix.applyBuildEnd(ctx, value) - if err != nil || !wrote { - return err // an entry that wrote nothing leaves the view as it is - } - return ix.keepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back + _, err := ix.applyBuildEnd(ctx, value) + return err // the imports reach the next take-in's revision (design 47 H §743, H-A-49) } // ApplyBuildEndsAtStart applies every entry of the build log read at start as ApplyBuildEnd does, and keeps the view @@ -63,20 +60,8 @@ func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, f wrote++ } } - var err error - if wrote > 0 { - err = ix.keepView(ctx) - } ix.logf("replayed %d build ends at start: %d wrote a module, in %s", len(values), wrote, time.Since(began).Round(time.Millisecond)) - return err -} - -// keepView is KeepView; a test replaces it to count the writes of the view. -func (ix *Indexer) keepView(ctx context.Context) error { - if ix.keptView != nil { - return ix.keptView(ctx) - } - return ix.KeepView(ctx) + return nil } // applyBuildEnd reports whether it wrote the module (its final write changed the held value). diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 456c2f48..4f308265 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -401,43 +401,3 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * } }) } - -// A start replays every build end of the build log: an entry that writes no module keeps no view, and the entries that -// write keep it once, after all of them (measured: the view kept per entry cost 50 min CPU per start). -func TestBuildEndsKeepTheViewOnlyWhenTheyWrite(t *testing.T) { - ctx := context.Background() - ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - if err := ix.Recompute(ctx); err != nil { - t.Fatal(err) - } - raw, _, _, _ := ix.Modules.Get(ctx, "avahi") - cur, err := meshcatalogclient.DecodeModule(raw) - if err != nil || cur.Fp2 == "" { - t.Fatalf("no fingerprint to start from: %v %q", err, cur.None2) - } - older := buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK) - writes := buildEnd(t, "avahi", cur.Fp2, meshSDK) - kept := 0 - ix.keptView = func(context.Context) error { kept++; return nil } - for _, c := range []struct { - name string - entries [][]byte - want int - }{ - {"no entry", nil, 0}, - {"entries that write nothing", [][]byte{older, []byte("not a build entry"), older}, 0}, - {"a write among entries that write nothing", [][]byte{older, writes, older}, 1}, - {"the same write again keeps nothing", [][]byte{writes, older}, 0}, - } { - kept = 0 - if err := ix.ApplyBuildEndsAtStart(ctx, c.entries, nil); err != nil || kept != c.want { - t.Errorf("%s: the view was kept %d times (%v), want %d", c.name, kept, err, c.want) - } - } - // The live path, one entry: the view is kept when it wrote. - kept = 0 - if err := ix.ApplyBuildEnd(ctx, older); err != nil || kept != 0 { - t.Errorf("a live entry that writes nothing kept the view %d times (%v)", kept, err) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index f871def7..3b585be1 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -270,7 +270,7 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t tool = got.(map[string]any) } } - return tool != nil && tool["name"] == "trunk.2", nil + return tool != nil && tool["name"] == "main.2", nil }}, "Outdated": {"", func(ix *Indexer) (bool, error) { a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{}) diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index ee7ece61..42f9e91e 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -279,10 +279,7 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ // message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is // and said: a failure never empties the bucket. func (ix *Indexer) Reconcile(ctx context.Context) error { - if err := ix.reconcile(ctx); err != nil { - return err - } - return ix.KeepView(ctx) // the modules dropped leave the view's next revision too + return ix.reconcile(ctx) // the modules dropped leave the next take-in's revision (design 47 H §743, H-A-49) } func (ix *Indexer) reconcile(ctx context.Context) error { diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 2fce27e8..151cfbae 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -180,18 +180,18 @@ func (ix *Indexer) clearPinMissing(ctx context.Context, key string) { // RevisionGroup says which watched branch's line a revision object belongs to, and its number; false for a candidate or an entry. func RevisionGroup(name string) (group string, revision int64, ok bool) { - if n, ok := TrunkRevisionOf(name); ok { - return "", n, true + if strings.HasPrefix(name, "candidate.") || strings.HasPrefix(name, "entry.") { + return "", 0, false } - if rest, ok := strings.CutPrefix(name, "branch."); ok { - i := strings.LastIndex(rest, ".") - if i > 0 { - if n, err := strconv.ParseInt(rest[i+1:], 10, 64); err == nil { - return rest[:i], n, true - } - } + i := strings.LastIndex(name, ".") + if i <= 0 { + return "", 0, false } - return "", 0, false + n, err := strconv.ParseInt(name[i+1:], 10, 64) + if err != nil || n <= 0 { + return "", 0, false + } + return name[:i], n, true } // CurrentRevisions are the kept revision objects that are current (§579): the newest of each watched branch, and every open @@ -474,22 +474,3 @@ func ConsumerTools(st Store) []stdio.Tool { Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}} } - -// PinPipelines pins the catalog graph revision a take-in made for the one pipeline the take-in starts, under the record -// `pipeline..` (the runner's key, design 47 H §743, A-449a), in the indexer's own pins state: so no later -// take-in removes the revision before the runner starts it. The pipeline's pipeline-started names the same record (a set), -// and its pipeline-ended removes it (§749). It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a -// source, an import) can make it non-current and remove it between the two; it answers that revision, 0 when none is kept. -func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64, entries []meshcatalog.TakeInEntry) int64 { - ix.keepMu.Lock() - defer ix.keepMu.Unlock() - revision := ix.NewestRevision(ctx, branch) - if ix.Pins == nil || revision <= 0 || len(entries) == 0 { - return revision - } - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))) - if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil { - ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err) - } - return revision -} diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index fb3172d6..d0ae03f6 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -44,35 +44,35 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } } reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} - keep := func(commit string) { + keep := func(commit string, source int64) { t.Helper() e := reader e.Commit = commit - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil { t.Fatal(err) } } - keep("c1") + keep("c1", 1) pending = 3 // pinned before the next keeps, which judge the revisions while the consumers are behind started("main.5", 1) started("main.5", 1) // heard twice: pins once started("main.6", 1) - keep("c2") - keep("c3") + keep("c2", 2) + keep("c3", 3) ix.JudgePins(ctx) - want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3") + want("while messages of the consumer are pending", "main.1", "main.2", "main.3") pending = 0 ix.JudgePins(ctx) - want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3") + want("pinned by two records, the middle one neither pinned nor current", "main.1", "main.3") if v, _, _, _ := ix.Pins.Get(ctx, "main.1"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5", "pipeline.main.6"}) { t.Errorf("the pin of main.1 holds %s, want the two pipelines' records once each", v) } ended("main.5", 1) ix.JudgePins(ctx) - want("one of two records freed it", "trunk.1", "trunk.3") + want("one of two records freed it", "main.1", "main.3") ended("main.6", 1) ix.JudgePins(ctx) - want("both freed it", "trunk.3") + want("both freed it", "main.3") if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v, want no key of the removed revision", keys) } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index a21bbd3c..419faf50 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -7,7 +7,6 @@ import ( "fmt" "regexp" "slices" - "strconv" "strings" "time" @@ -330,7 +329,7 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA case str(args, "group") != "" || str(args, "heads") != "": name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) case n > 0: - name = "trunk." + strconv.FormatInt(int64(n), 10) + name = BranchObject(TrunkBranch, int64(n)) default: newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go new file mode 100644 index 00000000..115b121c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -0,0 +1,117 @@ +package index + +import ( + "context" + "errors" + "fmt" + "maps" + "strconv" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrOldForm is a source-moved of the earlier form, one with no field its form requires (design 47 H §765, H-A-58): it is +// refused, said in the journal, and never taken in. +var ErrOldForm = errors.New("a source-moved of the old form") + +// TakenID is the duplicate id take-in.taken of a branch's source revision is said under, so a take-in said again is heard once. +func TakenID(branch string, source int64) string { + return takein.TakeInTakenEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) +} + +// TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each +// repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules +// registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch, +// ., carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and +// take-in.taken once, with branch, source, revision, call and one entry per mover. A revision already kept under that name (a +// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched +// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. +// An error leaves the source-moved held: ErrOldForm never takes it in, ErrObjectsFull waits for room (§744), any other is the +// failed read or write it says. +func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) { + taken := meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call} + if e.Branch == "" || e.Source <= 0 || e.Call == "" { + return taken, fmt.Errorf("%w: branch %q, source %d, call %q", ErrOldForm, e.Branch, e.Source, e.Call) + } + if ix.Revisions == nil { + return taken, errors.New("the indexer keeps no catalog revisions") + } + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + name := BranchObject(e.Branch, e.Source) + if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil { + taken.Revision, taken.Entries = kept.Revision, kept.Movers + ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers)) + return taken, ix.sayTaken(ctx, taken) + } else if !errors.Is(err, ErrNoObject) { + return taken, err + } + _, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch) + if err != nil { + return taken, fmt.Errorf("the newest catalog revision of %s: %w", e.Branch, err) + } + x := ix + if !isTrunkBranch(e.Branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, e.Branch) + if err != nil { + return taken, err + } + d := ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("take-in of "+e.Branch+": "+format, args...) } + x = d.ix + } + heads := map[string]string{} + var moved []string + for _, h := range e.Heads { + heads[h.Repository], moved = h.Commit, append(moved, h.Repository) + } + x.mu.Lock() + x.source = sourceNow{rev: uint64(e.Source), heads: mergedHeads(x.source.heads, heads)} + x.mu.Unlock() + for _, h := range e.Heads { + f, err := x.wholeAt(ctx, h.Repository, h.Commit) + if err != nil { + return taken, err + } + f.Branch = e.Branch + if _, err := x.ApplyMessage(ctx, f); err != nil { + return taken, fmt.Errorf("taking %s at %.12s in: %w", h.Repository, h.Commit, err) + } + } + entries, err := x.viewEntries(ctx) + if err != nil { + return taken, err + } + movers := moversOf(before, entries, moved) + if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil { + return taken, err + } + taken.Revision, taken.Entries = e.Source, movers + ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers)) + return taken, ix.sayTaken(ctx, taken) +} + +// mergedHeads is held with the heads given laid over it. +func mergedHeads(held, over map[string]string) map[string]string { + out := maps.Clone(held) + if out == nil { + out = map[string]string{} + } + maps.Copy(out, over) + return out +} + +// sayTaken says take-in.taken under its duplicate id; nil SayTakeIn says nothing. +func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) error { + if ix.SayTakeIn == nil { + return nil + } + if err := ix.SayTakeIn(ctx, takein.TakeInTakenEvent, TakenID(t.Branch, t.Source), t); err != nil { + return fmt.Errorf("take-in.taken of %s: %w", BranchObject(t.Branch, t.Source), err) + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go new file mode 100644 index 00000000..6d0d4e24 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -0,0 +1,94 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" +) + +// takeInIndexer is an indexer that takes source revisions in from the catalogue repository's trees at f's commit, keeping its +// revisions and pins, and the take-in events it says. +func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTaken, whole) { + t.Helper() + ix, _, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + objects := newMemObjects(1 << 22) + ix.Revisions, ix.Pins, ix.PinsPending = objects, newMem(), func(context.Context) (uint64, error) { return 0, nil } + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + if repository != f.Repository || sha != f.Merge { + return RepoRead{}, errors.New("no such commit") + } + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil } + said := &[]meshcatalog.TakeInTaken{} + ix.SayTakeIn = func(_ context.Context, event, id string, body any) error { + tk, ok := body.(meshcatalog.TakeInTaken) + if event != "take-in.taken" || !ok || id != TakenID(tk.Branch, tk.Source) { + t.Errorf("said %s %s %+v, want take-in.taken under its branch and source", event, id, body) + } + *said = append(*said, tk) + return nil + } + return ix, objects, said, f +} + +// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision . carrying its movers, pins it for the +// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover +// pins nothing and says no entries; the old form is refused. +func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { + ctx := context.Background() + ix, objects, said, f := takeInIndexer(t) + moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 7, Call: "0123456789abcdef", + Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}} + taken, err := ix.TakeInSource(ctx, moved) + if err != nil { + t.Fatal(err) + } + if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge || + taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { + t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken) + } + rev, _, err := ReadRevision(ctx, objects, "main.7") + if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { + t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err) + } + if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found { + t.Error("main.7 is not pinned for its pipeline") + } else { + var pv PinValue + if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) { + t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v) + } + } + puts := objects.puts + again, err := ix.TakeInSource(ctx, moved) + if err != nil || objects.puts != puts || !reflect.DeepEqual(again, taken) { + t.Errorf("a redelivery wrote %d objects and said %+v (%v), want none and the same", objects.puts-puts, again, err) + } + same := moved + same.Source, same.Call = 8, "fedcba9876543210" + none, err := ix.TakeInSource(ctx, same) + if err != nil || len(none.Entries) != 0 || none.Revision != 8 { + t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err) + } + if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found { + t.Error("a take-in with no mover pinned its revision") + } + if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil { + t.Errorf("main.8 is not kept: %v", err) + } + if len(*said) != 3 { + t.Errorf("said %d take-in.taken, want 3", len(*said)) + } + old := moved + old.Call = "" + if _, err := ix.TakeInSource(ctx, old); !errors.Is(err, ErrOldForm) { + t.Errorf("a source-moved with no call answered %v, want ErrOldForm", err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 40019e0f..4ebd5ce8 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -372,7 +372,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc if err := ix.writeGraph(ctx, edges); err != nil { return err } - return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges, ix.BuildModules())) + return nil // only a take-in keeps a catalog revision (design 47 H §743, H-A-49) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 83f6b032..6db6ed10 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -384,9 +384,9 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) { t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts) } - if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || + if back, got, err := ReadRevision(context.Background(), kept, "main.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || !slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) { - t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err) + t.Errorf("main.2 read back = %+v, %d entries, %v", back, len(got), err) } // W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its // entry gives it back; a bucket that keeps none is an empty view at 0. @@ -414,36 +414,31 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err) } _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) - if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { - t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) + if _, _, err := ReadRevision(context.Background(), kept, "main.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { + t.Errorf("main.2 with a lost entry read without error naming it: %v", err) } - // W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps - // none; a module leaving makes the next revision, naming it changed. + // Design 47 H §743, H-A-49: only a take-in keeps a catalog revision; a recompute keeps none and says no catalog-revision. ix.Revisions = newMemObjects(0) var revisions []meshcatalog.CatalogRevisionEvent ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { revisions = append(revisions, e) return nil } - if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" || - revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) { - t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err) + if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 0 { + t.Fatalf("a recompute said %+v (%v), want no revision", revisions, err) } graph := ix.Graph.(*memStore) written := graph.writes - if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 { - t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err) + if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { + t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err) } // A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute. delete(mods.vals, "other") if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } - if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) { - t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions) - } - if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) { - t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals)) + if names, _ := ix.Revisions.Names(context.Background()); len(names) != 0 { + t.Errorf("recomputes kept %v, want nothing", names) } if _, held := graph.vals["other"]; held { t.Error("graph.other is still held after its module went") @@ -506,11 +501,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if err := first.Apply(context.Background(), merged); err != nil { t.Fatal(err) } - if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || + takeIn := func(source int64) error { + entries, err := first.viewEntries(context.Background()) + if err != nil { + return err + } + return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil) + } + if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 || firstSaid[0].Entries != int64(len(builderModules(t))) { t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } - // KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision. + // A write to the view between take-ins (here a verdict carried into one module) is in the next take-in's revision. viewKey, _ := meshcatalogclient.ModulesKey("avahi") _ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) { m, _ := meshcatalogclient.DecodeModule(value) @@ -518,11 +520,11 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // data, err := meshcatalogclient.EncodeModule(m) return data, false, err }) - if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { - t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err) + if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { + t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err) } - if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { - t.Errorf("the view at a restart is read at %d (%v), want 2", at, err) + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 { + t.Errorf("the view at a restart is read at %d (%v), want 7", at, err) } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } @@ -538,21 +540,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("the first-view tools %+v serve no modules", FirstViewTools) } if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 2 || len(a.Values) != len(builderModules(t)) { - t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err) + a.Revision != 7 || len(a.Values) != len(builderModules(t)) { + t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err) } else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" { t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err) } - if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { - t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err) } // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { - t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) + a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err) } if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index f3d23fc8..cdc1fc3f 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -102,10 +102,10 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( n := NewNamed(newMem(), nil) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} n.Changed = ix.RemoveUnlisted - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } - if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil { // the revision that listed it is removed + if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed t.Fatal(err) } found := func() bool { @@ -213,10 +213,10 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin fp := namedFp("c") setup := func(n *Named) (*Indexer, func() bool) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} - if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } - if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil { + if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { t.Fatal(err) } return ix, func() bool {