mesh-delivery-indexer: a release pin only for a released mover, freed only by a newer release of it

A passed pipeline's movers include ones the assigner never takes (superseded, no version released,
a tier-late pipeline's later tiers); freeing their older release on the newer pipeline-wanted would
remove the revision their want still names, and a mover releasing nothing would pin a revision per
pipeline for good. The release record is now written only for a mover that released a version in
the pipeline, and pipeline-wanted frees a module's older record only where the newer pipeline's own
revision holds that module's release record.

Tests: a pins table row (a passed mover with no version pins nothing), the freeing sequence (a
passed pipeline releasing no version frees no older release, the next one that releases does), and
TestIndexerWhole now says a pipeline-wanted on the bus after main.12's passed end and waits for
main.11's release to be freed and main.11 removed, so the indexer's hearing of pipeline-wanted is
tested end to end (review round 1). No test added.

Builds: design 47 H-A-173 (hq design/47-H1-answers-59 21297155), H-A-170, H-A-168
This commit is contained in:
2026-11-20 17:58:45 +01:00
parent 9083287d57
commit 814244fd5a
3 changed files with 76 additions and 26 deletions
@@ -21,6 +21,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -340,7 +341,8 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende
return
}
raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source,
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}})
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1,
Version: "0.1." + s}}})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil {
t.Fatal(err)
}
@@ -371,8 +373,9 @@ func absent(v []byte) bool { return v == nil }
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
// (H-A-114, H-A-118); nothing takes it in on a
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
// room and takes main.13 in on it;
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and
// main.11 goes (H-A-168, H-A-170, H-A-173);
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
// the hand-act log before it is asked, and take-in.taken says it skipped;
// - a branch watched later is bound on registration-changed and its first revision taken in.
@@ -506,6 +509,19 @@ func TestIndexerWhole(t *testing.T) {
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
sayPipeline(t, ctx, js, true, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
var p index.PinValue
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilRevisionGone(t, ctx, js, "main.11")
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -85,9 +85,13 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
// A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed,
// one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none.
// one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run
// on request's pipeline, which delivers nothing, pin none.
if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" {
for _, m := range ended.Movers {
if m.Version == "" {
continue
}
if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil {
return err
}
@@ -109,9 +113,11 @@ func ReleaseRecord(branch, module string) string {
}
// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end,
// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names
// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A
// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing.
// tier-late included (design 47 H §751, H-A-168, H-A-170, H-A-173): a module whose release record the pipeline's own kept
// revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from
// every revision of the branch below it. A mover the revision holds no release of (superseded, no version released, a
// tier-late later tier) keeps its older release's revision. A pipeline whose revision is not kept frees nothing and says so;
// one applied twice changes nothing.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error {
if ix.Pins == nil || ix.Revisions == nil {
return errors.New("the indexer keeps no state " + PinsState)
@@ -125,6 +131,20 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision))
if err != nil {
return err
}
var held PinValue
if found {
_ = json.Unmarshal(own, &held)
}
var released []meshcatalog.TakeInEntry
for _, m := range rev.Movers {
if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) {
released = append(released, m)
}
}
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return err
@@ -134,7 +154,7 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc
if !ok || b != branch || r >= rev.Revision {
continue
}
for _, m := range rev.Movers {
for _, m := range released {
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return err
@@ -92,8 +92,9 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
}
// H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's
// pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes.
// H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose
// mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees
// the older release on its pipeline-wanted (its wants written to the end), and the revision goes.
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
take := func(source int64) {
t.Helper()
@@ -103,32 +104,39 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
t.Fatal(err)
}
}
end := func(source int64, outcome string) {
end := func(source int64, outcome, version string) {
t.Helper()
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}}
if err := ix.HeardPins(ctx, nil, &e); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
take(10)
end(10, runner.PipelineEndedOutcomePassed)
take(11)
end(11, runner.PipelineEndedOutcomeFailed)
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed)
want("two releases, the one between them gone", "main.10", "main.12")
for range 2 { // heard twice: changes nothing more
if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil {
wanted := func(source int64) {
t.Helper()
if err := ix.HeardPipelineWanted(ctx, "main", source); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
want("the newer release written everywhere", "main.12")
if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.12 holds %s, want its release alone", v)
take(10)
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
take(11)
end(11, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed, "")
wanted(12)
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
take(13)
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.10", "main.13")
wanted(13)
wanted(13) // heard twice: changes nothing more
want("the newer release written everywhere", "main.13")
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
}
}
@@ -158,6 +166,7 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
afterTheEnd []string
base int64 // the base main.5 names (H-A-154)
passed bool // main.5's pipeline passed: its release pins main.5
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
then []int64 // sources taken in with no mover after the end
released []string
}{
@@ -166,6 +175,8 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
} {
@@ -203,7 +214,10 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
}
}
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
if tc.unreleased {
end.Movers[0].Version = ""
}
if tc.passed {
end.Outcome = runner.PipelineEndedOutcomePassed
}