mesh-delivery-indexer: a take-in pins the branch's previous revision for its pipeline, so validate's base stays

The pin judge removed the branch's previous revision in the very write that took the next one in
(pipelines main.13 and main.15 on 2026-11-20), so the validate step's case before could not be read
and the composition was never asked. The take-in now pins the newest revision it kept before its
write under the same pipeline record as its own, and the pipeline's end frees the record on every
key it pins; a take-in with no revision before it pins only its own and says so.

Builds: H-A-147 (hq design/47-H1-answers-59 310f8955ba16a1ca8ae9661d8196cab16e3d9597), design 47 H §743, §749, H-A-108 (3)
This commit is contained in:
2026-11-20 16:27:21 +01:00
parent 8167d825ed
commit e5e5c50e67
4 changed files with 121 additions and 12 deletions
@@ -361,11 +361,13 @@ func absent(v []byte) bool { return v == nil }
//
// - main's first source revision (start true) is taken in with no mover and pins nothing; the next, moving avahi's tree, is one
// revision main.11 carrying avahi as its one mover, pinned pipeline.main.11, said once as take-in.taken; the same source-moved
// given again says the same from the kept movers and keeps no second revision;
// - pipeline-started of main.12 pins it, a restart keeps main.11 while its pipeline runs and removes main.10;
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
// (H-A-114, H-A-118); nothing takes it in on a
// timer: main.11's pipeline-ended frees its pin, the revision goes, the indexer says its room and takes main.13 in on it;
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
// room and takes main.13 in on it;
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
// the hand-act log before it is asked, and take-in.taken says it skipped;
// - a branch watched later is bound on registration-changed and its first revision taken in.
@@ -429,6 +431,7 @@ func TestIndexerWhole(t *testing.T) {
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
info, err := objects.open(ctx)
if err != nil {
t.Fatal(err)
@@ -445,8 +448,8 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
}
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it, and the judge at the start removes
// main.10, which nothing pins and which is no longer current.
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it and main.10, its validate base
// (H-A-147), which the take-in pinned for it.
sayMoved(t, ctx, js, "main", 12, c12, false)
heard.next(t, "main", 12)
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
@@ -455,9 +458,10 @@ func TestIndexerWhole(t *testing.T) {
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
restart()
untilRevisionGone(t, ctx, js, "main.10")
if _, _, err := index.ReadRevision(ctx, objects, "main.11"); err != nil {
t.Fatalf("main.11 is not kept across a restart while its pipeline runs: %v", err)
for _, name := range []string{"main.10", "main.11"} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
}
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
@@ -487,7 +491,8 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
}
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -268,7 +268,7 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
// pipeline.<branch>.<src>, in the same write when there is one (A-449a). A revision already kept under that name is not written
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
@@ -330,6 +330,16 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
}
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
if newest.Revision > 0 && newest.Revision != src {
if err := ix.pin(ctx, PinKey(branch, newest.Revision), record); err != nil {
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
}
} else {
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
}
}
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
if removed, err := ix.judgePinsLocked(ctx); err != nil {
@@ -72,7 +72,8 @@ func pinObject(key string) (string, bool) {
}
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it. A body
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
// other revision the record pins (H-A-147). A body
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
// again; one applied twice changes nothing.
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
@@ -83,7 +84,34 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
case started != nil && started.Revision > 0 && started.Pipeline != "":
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline))
record := PipelineRecord(ended.Pipeline)
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
return err
}
return ix.unpinElsewhere(ctx, record)
}
return nil
}
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
// same pipeline (H-A-147), which the pipeline-ended does not name.
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return err
}
for _, k := range pinKeys {
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return err
}
var pv PinValue
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
continue
}
if err := ix.unpin(ctx, k, record); err != nil {
return err
}
}
return nil
}
@@ -3,6 +3,7 @@ package index
import (
"context"
"encoding/json"
"fmt"
"slices"
"testing"
"time"
@@ -102,3 +103,68 @@ func mustRecords(t *testing.T, v []byte) []string {
slices.Sort(pv.Records)
return pv.Records
}
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
// runs; a take-in with no revision before it pins only its own; the pipeline's end frees both.
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
ctx := context.Background()
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
for _, tc := range []struct {
name string
before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string
afterTheEnd []string
}{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
} {
t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
Now: func() time.Time { return time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) },
PinsPending: func(context.Context) (uint64, error) { return 0, nil }}
keep := func(source int64, movers []meshcatalog.TakeInEntry) {
t.Helper()
e := reader
e.Commit = fmt.Sprintf("c%d", source)
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, movers); err != nil {
t.Fatal(err)
}
}
kept := func() []string {
n, _ := ix.Revisions.Names(ctx)
n = slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
slices.Sort(n)
return n
}
for _, s := range tc.before {
keep(s, nil)
}
keep(5, mover)
if got := kept(); !slices.Equal(got, tc.whileRuns) {
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
}
for _, k := range tc.whileRuns {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
}
}
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
}
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
for _, k := range keys {
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
}
}
}
})
}
}