diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 7086bbc10..eb29829b2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -73,8 +73,8 @@ func pinObject(key string) (string, bool) { } // HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a -// pipeline-started pins the revision it names under the record `pipeline.`, its pipeline-ended frees it and every -// other revision the record pins (H-A-147). A body +// pipeline-started pins the revision it names, and each revision its movers are kept at (design 47 H2 H-A-496), under the +// record `pipeline.`, its pipeline-ended frees it and every other revision the record pins (H-A-147). A body // that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it // again; one applied twice changes nothing. func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error { @@ -83,7 +83,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte } switch { case started != nil && started.Revision > 0 && started.Pipeline != "": - if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil { + record := PipelineRecord(started.Pipeline) + if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), record); err != nil { + return err + } + // Every revision the pipeline's movers are kept at is pinned under its record too, so no judge removes one it still + // reads (design 47 H2 H-A-496); the latest started's kept set is the pipeline's kept pins, so a kept revision the + // record pins that this started no longer names is freed (H-A-496 addendum). A kept revision is always newer than the + // pipeline's own (a re-pin of its own source, H-A-283), so its own revision and its validate base stay pinned. + if err := ix.pinKept(ctx, started.Branch, started.Revision, started.Kept, record); err != nil { return err } // The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)). @@ -229,6 +237,41 @@ func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error { return nil } +// pinKept pins each of kept, the revisions a pipeline of branch at revision own keeps movers at, under record, and frees +// every revision newer than own the record pins that kept does not name (design 47 H2 H-A-496 and its addendum). +func (ix *Indexer) pinKept(ctx context.Context, branch string, own int64, kept []int64, record string) error { + for _, r := range kept { + if r <= 0 || r == own { + continue + } + if err := ix.pin(ctx, PinKey(branch, r), record); err != nil { + return err + } + } + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return err + } + for _, k := range pinKeys { + b, r, ok := ParsePinKey(k) + if !ok || b != branch || r <= own || slices.Contains(kept, r) { + continue + } + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return err + } + var pv PinValue + if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) { + continue + } + if err := ix.unpin(ctx, k, record); err != nil { + return err + } + } + return nil +} + // PipelineRecord is the record a pipeline pins its revision under: `pipeline..` (design 47 H §743, A-449a). func PipelineRecord(pipeline string) string { return "pipeline." + pipeline } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index c6de51474..f8c203b0b 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -13,6 +13,47 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) +// Design 47 H2 H-A-496 and its addendum: a pipeline-started pins the pipeline's own revision and each revision its movers +// are kept at under the pipeline's record; a later started (said again at a re-pin) is the pipeline's kept set, freeing a +// kept revision it no longer names but never the pipeline's own; pipeline-ended frees them all. +func TestAPipelinesKeptRevisionsArePinnedUntilItEnds(t *testing.T) { + ctx := context.Background() + pending := uint64(1) // the keeps judge nothing while the consumer is behind + ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), + Now: func() time.Time { return time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }} + kept := func() []string { + n, _ := ix.Revisions.Names(ctx) + return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok }) + } + reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} + for source := int64(1); source <= 6; source++ { + e := reader + e.Commit = fmt.Sprintf("c%d", source) + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil { + t.Fatal(err) + } + } + pending = 0 + heard := func(started *runner.PipelineStarted, ended *runner.PipelineEnded, want ...string) { + t.Helper() + if err := ix.HeardPins(ctx, started, ended); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + if got := kept(); !slices.Equal(got, want) { + t.Errorf("kept %v, want %v", got, want) + } + } + heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{2, 3, 4}}, nil, + "main.1", "main.2", "main.3", "main.4", "main.6") + if v, _, _, _ := ix.Pins.Get(ctx, "main.3"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.1"}) { + t.Errorf("the pin of main.3 holds %s, want the pipeline's record", v) + } + heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{3, 4}}, nil, + "main.1", "main.3", "main.4", "main.6") + heard(nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.1", Revision: 1, Outcome: runner.PipelineEndedOutcomeFailed}, "main.6") +} + // Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its // pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins // its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor