Merge pull request 'indexer: a take-in pins the branch's previous revision for its pipeline, so validate's base stays (H-A-147)' (#271) from fix/47-H1-validate-previous-revision into main

This commit was merged in pull request #271.
This commit is contained in:
2026-11-20 15:31:05 +00:00
4 changed files with 121 additions and 12 deletions
@@ -361,11 +361,13 @@ func absent(v []byte) bool { return v == nil }
//
// - main's first source revision (start true) is taken in with no mover and pins nothing; the next, moving avahi's tree, is one
// revision main.11 carrying avahi as its one mover, pinned pipeline.main.11, said once as take-in.taken; the same source-moved
// given again says the same from the kept movers and keeps no second revision;
// - pipeline-started of main.12 pins it, a restart keeps main.11 while its pipeline runs and removes main.10;
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
// (H-A-114, H-A-118); nothing takes it in on a
// timer: main.11's pipeline-ended frees its pin, the revision goes, the indexer says its room and takes main.13 in on it;
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
// room and takes main.13 in on it;
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
// the hand-act log before it is asked, and take-in.taken says it skipped;
// - a branch watched later is bound on registration-changed and its first revision taken in.
@@ -429,6 +431,7 @@ func TestIndexerWhole(t *testing.T) {
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
info, err := objects.open(ctx)
if err != nil {
t.Fatal(err)
@@ -445,8 +448,8 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
}
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it, and the judge at the start removes
// main.10, which nothing pins and which is no longer current.
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it and main.10, its validate base
// (H-A-147), which the take-in pinned for it.
sayMoved(t, ctx, js, "main", 12, c12, false)
heard.next(t, "main", 12)
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
@@ -455,9 +458,10 @@ func TestIndexerWhole(t *testing.T) {
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
restart()
untilRevisionGone(t, ctx, js, "main.10")
if _, _, err := index.ReadRevision(ctx, objects, "main.11"); err != nil {
t.Fatalf("main.11 is not kept across a restart while its pipeline runs: %v", err)
for _, name := range []string{"main.10", "main.11"} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
}
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
@@ -487,7 +491,8 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
}
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -268,7 +268,7 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
// pipeline.<branch>.<src>, in the same write when there is one (A-449a). A revision already kept under that name is not written
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
@@ -330,6 +330,16 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
}
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
if newest.Revision > 0 && newest.Revision != src {
if err := ix.pin(ctx, PinKey(branch, newest.Revision), record); err != nil {
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
}
} else {
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
}
}
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
if removed, err := ix.judgePinsLocked(ctx); err != nil {
@@ -72,7 +72,8 @@ func pinObject(key string) (string, bool) {
}
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it. A body
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
// other revision the record pins (H-A-147). A body
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
// again; one applied twice changes nothing.
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
@@ -83,7 +84,34 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
case started != nil && started.Revision > 0 && started.Pipeline != "":
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline))
record := PipelineRecord(ended.Pipeline)
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
return err
}
return ix.unpinElsewhere(ctx, record)
}
return nil
}
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
// same pipeline (H-A-147), which the pipeline-ended does not name.
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return err
}
for _, k := range pinKeys {
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return err
}
var pv PinValue
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
continue
}
if err := ix.unpin(ctx, k, record); err != nil {
return err
}
}
return nil
}
@@ -3,6 +3,7 @@ package index
import (
"context"
"encoding/json"
"fmt"
"slices"
"testing"
"time"
@@ -102,3 +103,68 @@ func mustRecords(t *testing.T, v []byte) []string {
slices.Sort(pv.Records)
return pv.Records
}
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
// runs; a take-in with no revision before it pins only its own; the pipeline's end frees both.
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
ctx := context.Background()
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
for _, tc := range []struct {
name string
before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string
afterTheEnd []string
}{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
} {
t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
Now: func() time.Time { return time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) },
PinsPending: func(context.Context) (uint64, error) { return 0, nil }}
keep := func(source int64, movers []meshcatalog.TakeInEntry) {
t.Helper()
e := reader
e.Commit = fmt.Sprintf("c%d", source)
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, movers); err != nil {
t.Fatal(err)
}
}
kept := func() []string {
n, _ := ix.Revisions.Names(ctx)
n = slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
slices.Sort(n)
return n
}
for _, s := range tc.before {
keep(s, nil)
}
keep(5, mover)
if got := kept(); !slices.Equal(got, tc.whileRuns) {
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
}
for _, k := range tc.whileRuns {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
}
}
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
}
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
for _, k := range keys {
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
}
}
}
})
}
}