diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 22c71590..cfcc1b8f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -340,7 +341,8 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende return } raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source, - Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}}) + Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1, + Version: "0.1." + s}}}) if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil { t.Fatal(err) } @@ -371,8 +373,9 @@ func absent(v []byte) bool { return v == nil } // - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // (H-A-114, H-A-118); nothing takes it in on a -// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its -// room and takes main.13 in on it; +// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the +// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and +// main.11 goes (H-A-168, H-A-170, H-A-173); // - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in // the hand-act log before it is asked, and take-in.taken says it skipped; // - a branch watched later is bound on registration-changed and its first revision taken in. @@ -506,6 +509,19 @@ func TestIndexerWhole(t *testing.T) { // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), + // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). + sayPipeline(t, ctx, js, true, "main", 12) + untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + var p index.PinValue + return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) + }) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { + t.Fatal(err) + } + untilKey(t, ctx, pins, index.PinKey("main", 11), absent) + untilRevisionGone(t, ctx, js, "main.11") if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 9b8117bc..e9111d80 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -85,9 +85,13 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) case ended != nil && ended.Revision > 0 && ended.Pipeline != "": // A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed, - // one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none. + // one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run + // on request's pipeline, which delivers nothing, pin none. if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" { for _, m := range ended.Movers { + if m.Version == "" { + continue + } if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil { return err } @@ -109,9 +113,11 @@ func ReleaseRecord(branch, module string) string { } // HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end, -// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names -// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A -// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing. +// tier-late included (design 47 H §751, H-A-168, H-A-170, H-A-173): a module whose release record the pipeline's own kept +// revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from +// every revision of the branch below it. A mover the revision holds no release of (superseded, no version released, a +// tier-late later tier) keeps its older release's revision. A pipeline whose revision is not kept frees nothing and says so; +// one applied twice changes nothing. func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) @@ -125,6 +131,20 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc if err != nil { return fmt.Errorf("%s: %w", name, err) } + own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision)) + if err != nil { + return err + } + var held PinValue + if found { + _ = json.Unmarshal(own, &held) + } + var released []meshcatalog.TakeInEntry + for _, m := range rev.Movers { + if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) { + released = append(released, m) + } + } pinKeys, err := ix.Pins.Keys(ctx) if err != nil { return err @@ -134,7 +154,7 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc if !ok || b != branch || r >= rev.Revision { continue } - for _, m := range rev.Movers { + for _, m := range released { v, _, found, err := ix.Pins.Get(ctx, k) if err != nil { return err diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index b2e7efa6..24ac07f4 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -92,8 +92,9 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v after the pin drop, want no main.1", keys) } - // H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's - // pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes. + // H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose + // mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees + // the older release on its pipeline-wanted (its wants written to the end), and the revision goes. mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} take := func(source int64) { t.Helper() @@ -103,32 +104,39 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Fatal(err) } } - end := func(source int64, outcome string) { + end := func(source int64, outcome, version string) { t.Helper() e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, - Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} if err := ix.HeardPins(ctx, nil, &e); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - take(10) - end(10, runner.PipelineEndedOutcomePassed) - take(11) - end(11, runner.PipelineEndedOutcomeFailed) - want("a passed release past a later failed pipeline", "main.10", "main.11") - take(12) - end(12, runner.PipelineEndedOutcomePassed) - want("two releases, the one between them gone", "main.10", "main.12") - for range 2 { // heard twice: changes nothing more - if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil { + wanted := func(source int64) { + t.Helper() + if err := ix.HeardPipelineWanted(ctx, "main", source); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - want("the newer release written everywhere", "main.12") - if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { - t.Errorf("the pin of main.12 holds %s, want its release alone", v) + take(10) + end(10, runner.PipelineEndedOutcomePassed, "0.1.10") + take(11) + end(11, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.10", "main.11") + take(12) + end(12, runner.PipelineEndedOutcomePassed, "") + wanted(12) + want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") + take(13) + end(13, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.10", "main.13") + wanted(13) + wanted(13) // heard twice: changes nothing more + want("the newer release written everywhere", "main.13") + if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.13 holds %s, want its release alone", v) } } @@ -158,6 +166,7 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { afterTheEnd []string base int64 // the base main.5 names (H-A-154) passed bool // main.5's pipeline passed: its release pins main.5 + unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) then []int64 // sources taken in with no mover after the end released []string }{ @@ -166,6 +175,8 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, } { @@ -203,7 +214,10 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { } } end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, - Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} + if tc.unreleased { + end.Movers[0].Version = "" + } if tc.passed { end.Outcome = runner.PipelineEndedOutcomePassed }