H-A-496 indexer: pin the revisions a pipeline's movers are kept at
The indexer pinned only a pipeline's own revision, so JudgePins removed revision 292 while main.291's kept mover still had to read it. HeardPins now pins every revision pipeline-started names in kept under the pipeline's record, takes the latest started (said again at a re-pin) as the kept set, freeing a kept revision it no longer names but never the pipeline's own or its validate base, and pipeline-ended frees them all. JudgePins is unchanged. Builds: design 47 H2 H-A-496, H-A-496 addendum
This commit is contained in:
@@ -73,8 +73,8 @@ func pinObject(key string) (string, bool) {
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
|
||||
// other revision the record pins (H-A-147). A body
|
||||
// pipeline-started pins the revision it names, and each revision its movers are kept at (design 47 H2 H-A-496), under the
|
||||
// record `pipeline.<pipeline>`, its pipeline-ended frees it and every other revision the record pins (H-A-147). A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
@@ -83,7 +83,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
switch {
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil {
|
||||
record := PipelineRecord(started.Pipeline)
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), record); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every revision the pipeline's movers are kept at is pinned under its record too, so no judge removes one it still
|
||||
// reads (design 47 H2 H-A-496); the latest started's kept set is the pipeline's kept pins, so a kept revision the
|
||||
// record pins that this started no longer names is freed (H-A-496 addendum). A kept revision is always newer than the
|
||||
// pipeline's own (a re-pin of its own source, H-A-283), so its own revision and its validate base stay pinned.
|
||||
if err := ix.pinKept(ctx, started.Branch, started.Revision, started.Kept, record); err != nil {
|
||||
return err
|
||||
}
|
||||
// The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)).
|
||||
@@ -229,6 +237,41 @@ func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinKept pins each of kept, the revisions a pipeline of branch at revision own keeps movers at, under record, and frees
|
||||
// every revision newer than own the record pins that kept does not name (design 47 H2 H-A-496 and its addendum).
|
||||
func (ix *Indexer) pinKept(ctx context.Context, branch string, own int64, kept []int64, record string) error {
|
||||
for _, r := range kept {
|
||||
if r <= 0 || r == own {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(branch, r), record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r <= own || slices.Contains(kept, r) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
|
||||
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
|
||||
|
||||
|
||||
@@ -13,6 +13,47 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-496 and its addendum: a pipeline-started pins the pipeline's own revision and each revision its movers
|
||||
// are kept at under the pipeline's record; a later started (said again at a re-pin) is the pipeline's kept set, freeing a
|
||||
// kept revision it no longer names but never the pipeline's own; pipeline-ended frees them all.
|
||||
func TestAPipelinesKeptRevisionsArePinnedUntilItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pending := uint64(1) // the keeps judge nothing while the consumer is behind
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
}
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
for source := int64(1); source <= 6; source++ {
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("c%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pending = 0
|
||||
heard := func(started *runner.PipelineStarted, ended *runner.PipelineEnded, want ...string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPins(ctx, started, ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
if got := kept(); !slices.Equal(got, want) {
|
||||
t.Errorf("kept %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{2, 3, 4}}, nil,
|
||||
"main.1", "main.2", "main.3", "main.4", "main.6")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.3"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.1"}) {
|
||||
t.Errorf("the pin of main.3 holds %s, want the pipeline's record", v)
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{3, 4}}, nil,
|
||||
"main.1", "main.3", "main.4", "main.6")
|
||||
heard(nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.1", Revision: 1, Outcome: runner.PipelineEndedOutcomeFailed}, "main.6")
|
||||
}
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
|
||||
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
|
||||
|
||||
Reference in New Issue
Block a user