diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 9c2dbddd..889e7cdc 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -361,11 +361,13 @@ func absent(v []byte) bool { return v == nil } // // - main's first source revision (start true) is taken in with no mover and pins nothing; the next, moving avahi's tree, is one // revision main.11 carrying avahi as its one mover, pinned pipeline.main.11, said once as take-in.taken; the same source-moved -// given again says the same from the kept movers and keeps no second revision; -// - pipeline-started of main.12 pins it, a restart keeps main.11 while its pipeline runs and removes main.10; +// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is +// pinned pipeline.main.11 in the same write as validate's base (H-A-147); +// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // (H-A-114, H-A-118); nothing takes it in on a -// timer: main.11's pipeline-ended frees its pin, the revision goes, the indexer says its room and takes main.13 in on it; +// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its +// room and takes main.13 in on it; // - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in // the hand-act log before it is asked, and take-in.taken says it skipped; // - a branch watched later is bound on registration-changed and its first revision taken in. @@ -429,6 +431,7 @@ func TestIndexerWhole(t *testing.T) { t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err) } untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11")) + untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147 info, err := objects.open(ctx) if err != nil { t.Fatal(err) @@ -445,8 +448,8 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err) } - // pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it, and the judge at the start removes - // main.10, which nothing pins and which is no longer current. + // pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it and main.10, its validate base + // (H-A-147), which the take-in pinned for it. sayMoved(t, ctx, js, "main", 12, c12, false) heard.next(t, "main", 12) if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below @@ -455,9 +458,10 @@ func TestIndexerWhole(t *testing.T) { sayPipeline(t, ctx, js, false, "main", 12) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) restart() - untilRevisionGone(t, ctx, js, "main.10") - if _, _, err := index.ReadRevision(ctx, objects, "main.11"); err != nil { - t.Fatalf("main.11 is not kept across a restart while its pipeline runs: %v", err) + for _, name := range []string{"main.10", "main.11"} { + if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { + t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) + } } // The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it. @@ -487,7 +491,8 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) - untilKey(t, ctx, pins, index.PinKey("main", 11), absent) + untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147) + untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index d340e193..b101c8ea 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -268,7 +268,7 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata // KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design // 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the // branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, -// pipeline.., in the same write when there is one (A-449a). A revision already kept under that name is not written +// pipeline.., in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written // again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { return ix.keepBranchEntries(ctx, branch, src, entries, movers) @@ -330,6 +330,16 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) } + // The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of + // the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it. + if newest.Revision > 0 && newest.Revision != src { + if err := ix.pin(ctx, PinKey(branch, newest.Revision), record); err != nil { + ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err) + return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) + } + } else { + ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record) + } } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. if removed, err := ix.judgePinsLocked(ctx); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 439c97c2..5fa6c844 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -72,7 +72,8 @@ func pinObject(key string) (string, bool) { } // HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a -// pipeline-started pins the revision it names under the record `pipeline.`, its pipeline-ended frees it. A body +// pipeline-started pins the revision it names under the record `pipeline.`, its pipeline-ended frees it and every +// other revision the record pins (H-A-147). A body // that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it // again; one applied twice changes nothing. func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error { @@ -83,7 +84,34 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte case started != nil && started.Revision > 0 && started.Pipeline != "": return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) case ended != nil && ended.Revision > 0 && ended.Pipeline != "": - return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline)) + record := PipelineRecord(ended.Pipeline) + if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil { + return err + } + return ix.unpinElsewhere(ctx, record) + } + return nil +} + +// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the +// same pipeline (H-A-147), which the pipeline-ended does not name. +func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error { + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return err + } + for _, k := range pinKeys { + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return err + } + var pv PinValue + if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) { + continue + } + if err := ix.unpin(ctx, k, record); err != nil { + return err + } } return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 1c48eef8..4bcc9dab 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "fmt" "slices" "testing" "time" @@ -102,3 +103,68 @@ func mustRecords(t *testing.T, v []byte) []string { slices.Sort(pv.Records) return pv.Records } + +// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept +// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline +// runs; a take-in with no revision before it pins only its own; the pipeline's end frees both. +func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { + ctx := context.Background() + reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} + mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} + for _, tc := range []struct { + name string + before []int64 // sources taken in with no mover before the take-in of source 5 + whileRuns []string + afterTheEnd []string + }{ + {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}}, + {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}}, + {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, + } { + t.Run(tc.name, func(t *testing.T) { + ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), + Now: func() time.Time { return time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) }, + PinsPending: func(context.Context) (uint64, error) { return 0, nil }} + keep := func(source int64, movers []meshcatalog.TakeInEntry) { + t.Helper() + e := reader + e.Commit = fmt.Sprintf("c%d", source) + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, movers); err != nil { + t.Fatal(err) + } + } + kept := func() []string { + n, _ := ix.Revisions.Names(ctx) + n = slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok }) + slices.Sort(n) + return n + } + for _, s := range tc.before { + keep(s, nil) + } + keep(5, mover) + if got := kept(); !slices.Equal(got, tc.whileRuns) { + t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns) + } + for _, k := range tc.whileRuns { + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) { + t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) + } + } + if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + if got := kept(); !slices.Equal(got, tc.afterTheEnd) { + t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd) + } + if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 { + for _, k := range keys { + if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 { + t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v) + } + } + } + }) + } +}