From 9083287d57fade9802852a7d470c3327413471ee Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:50:01 +0100 Subject: [PATCH 1/3] mesh-delivery-indexer: a passed pipeline's release pins its revision until a newer one is the want MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin judge removed main.23 and its entries when main.24 became current, while the assigner had just written main.23's builder as the want: the node-engine asked its entry and was told none is kept, holding the want for good. A passed pipeline-ended now pins the pipeline's kept revision under a record release.., one per mover, beside the pipeline's own pin; the seat mesh-wants' pipeline-wanted of a newer pipeline on that branch (its wants written to the end, tier-late included) frees that record from every revision below the newer one, for each module the newer revision's movers name. The indexer consumes pipeline-wanted (its manifest says so). A revision a release pin holds keeps its entries, as removeUnlisted already does for any kept revision. Tests: rows in the pins table (a passed pipeline's revision kept once no longer current; a failed one's goes), the freeing in the pins test (a release past a failed pipeline, freed by the next passed one's pipeline-wanted, twice changing nothing), and a pipeline-wanted row in the events test. TestIndexerWhole now reads main.11's release beside its validate base, and waits for the held source-moved's acknowledgement before it reads the consumer (the write comes first). No test added. Builds: design 47 H-A-168, H-A-170 (hq design/47-H1-answers-59 87b43ec8), H-A-147, H §749 --- .../mesh-delivery-indexer/integration_test.go | 7 +- .../cmd/mesh-delivery-indexer/named_test.go | 22 ++++-- .../cmd/mesh-delivery-indexer/pins.go | 12 ++-- .../cmd/mesh-delivery-indexer/runs.go | 23 ++++++- .../takein_integration_test.go | 20 ++++-- .../internal/index/pins.go | 63 ++++++++++++++++- .../internal/index/pins_test.go | 68 +++++++++++++++++-- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 9 files changed, 190 insertions(+), 28 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 88112068..745cfab5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -303,8 +303,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) // The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register - // (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed - // and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them. + // (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed, + // the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted + // (H-A-170), as the grants part makes them. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil { t.Fatal(err) } @@ -313,7 +314,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged, meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll, - meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} { + meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index ab3faf53..7fafd036 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 // or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked. for _, row := range []struct { name, body string - ended bool + ended, wanted bool err error acked, termd int started, endedOf string }{ {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, + {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1}, {name: "not json", body: `nope`, termd: 1}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, } { ix := &fakeRuns{err: row.err} m := &fakeMsg{subject: "x", data: []byte(row.body)} - if row.ended { + switch { + case row.wanted: + heardPipelineWanted(context.Background(), ix, m, t.Logf) + if ix.wanted != "main.12" { + t.Errorf("%s: freed the releases below %q, want main.12", row.name, ix.wanted) + } + case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) - } else { + default: heardRunStarted(context.Background(), ix, m, t.Logf) } if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf { @@ -139,8 +146,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { // fakeRuns records the run events handed to it and answers err. type fakeRuns struct { - err error - started, ended string + err error + started, ended, wanted string } func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error { @@ -157,4 +164,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64) error { + f.wanted = fmt.Sprintf("%s.%d", branch, source) + return f.err +} + func (f *fakeRuns) JudgePins(context.Context) {} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go index 49f9132f..a5104616 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go @@ -6,15 +6,17 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" ) -// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on -// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, -// and only then acknowledged, so one said while the indexer was down is applied when it comes back. +// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release +// pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable +// consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then +// acknowledged, so one said while the indexer was down is applied when it comes back. -// pinsStreams are the streams of the runner's events the indexer's pins follow. -var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded} +// pinsStreams are the streams of the events the indexer's pins follow. +var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted} // pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet. func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index 9a3d42b5..033be1c4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -8,17 +8,20 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) // runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline), -// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins). +// and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted, +// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins). type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error + HeardPipelineWanted(ctx context.Context, branch string, source int64) error JudgePins(ctx context.Context) } @@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, {meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }}, + {meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }}, } { _ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error { cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf) @@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func ix.JudgePins(ctx) } +// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late +// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168, +// H-A-170). +func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { + var e meshwants.PipelineWanted + if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 { + logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source) + settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) + if err == nil { + ix.JudgePins(ctx) + } +} + func settleRun(m delivered, err error, what string, logf func(string, ...any)) { switch { case err == nil: diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 98cc851a..22c71590 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "os" + "slices" "strconv" "strings" "sync" @@ -345,11 +346,15 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende } } -// pinned is whether a pins value names the record. -func pinned(record string) func([]byte) bool { +// pinned is whether a pins value names exactly the records, in any order. +func pinned(records ...string) func([]byte) bool { return func(v []byte) bool { var p index.PinValue - return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record + if v == nil || json.Unmarshal(v, &p) != nil { + return false + } + slices.Sort(p.Records) + return slices.Equal(p.Records, slices.Sorted(slices.Values(records))) } } @@ -484,7 +489,9 @@ func TestIndexerWhole(t *testing.T) { return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() && hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13 }) - // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held. + // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the + // write, so it is waited for before the consumer is read. + heard.acked(t, "main") if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil { t.Fatal(err) } else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 { @@ -496,8 +503,9 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147) - untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). + untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) + untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 5fa6c844..9b8117bc 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -20,8 +20,8 @@ import ( // The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153, // §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `.` (the watched branch -// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each -// running pipeline's (design 47 H §743, §749). +// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each +// running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170). const ( // PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB). PinsState = "pins" @@ -84,6 +84,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte case started != nil && started.Revision > 0 && started.Pipeline != "": return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) case ended != nil && ended.Revision > 0 && ended.Pipeline != "": + // A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed, + // one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none. + if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" { + for _, m := range ended.Movers { + if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil { + return err + } + } + } record := PipelineRecord(ended.Pipeline) if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil { return err @@ -93,6 +102,56 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte return nil } +// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release..`, +// each name escaped as a key's name is (design 47 H-A-168, H-A-170). +func ReleaseRecord(branch, module string) string { + return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module) +} + +// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end, +// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names +// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A +// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error { + if ix.Pins == nil || ix.Revisions == nil { + return errors.New("the indexer keeps no state " + PinsState) + } + name, err := revisionOfSource(ctx, ix.Revisions, branch, source) + if err != nil { + ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) + return nil + } + rev, _, err := ReadRevision(ctx, ix.Revisions, name) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return err + } + for _, k := range pinKeys { + b, r, ok := ParsePinKey(k) + if !ok || b != branch || r >= rev.Revision { + continue + } + for _, m := range rev.Movers { + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return err + } + var pv PinValue + record := ReleaseRecord(branch, m.Module) + if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) { + continue + } + if err := ix.unpin(ctx, k, record); err != nil { + return err + } + } + } + return nil +} + // unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the // same pipeline (H-A-147), which the pipeline-ended does not name. func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index a2f89cc6..b2e7efa6 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -92,6 +92,44 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v after the pin drop, want no main.1", keys) } + // H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's + // pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes. + mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} + take := func(source int64) { + t.Helper() + e := reader + e.Commit = fmt.Sprintf("r%d", source) + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil { + t.Fatal(err) + } + } + end := func(source int64, outcome string) { + t.Helper() + e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + if err := ix.HeardPins(ctx, nil, &e); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + } + take(10) + end(10, runner.PipelineEndedOutcomePassed) + take(11) + end(11, runner.PipelineEndedOutcomeFailed) + want("a passed release past a later failed pipeline", "main.10", "main.11") + take(12) + end(12, runner.PipelineEndedOutcomePassed) + want("two releases, the one between them gone", "main.10", "main.12") + for range 2 { // heard twice: changes nothing more + if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + } + want("the newer release written everywhere", "main.12") + if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.12 holds %s, want its release alone", v) + } } func mustRecords(t *testing.T, v []byte) []string { @@ -107,7 +145,8 @@ func mustRecords(t *testing.T, v []byte) []string { // H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept // before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline // runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only -// its own and names no base; the pipeline's end frees both. +// its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release +// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current. func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { ctx := context.Background() reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} @@ -117,11 +156,18 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string - base int64 // the base main.5 names (H-A-154) + base int64 // the base main.5 names (H-A-154) + passed bool // main.5's pipeline passed: its release pins main.5 + then []int64 // sources taken in with no mover after the end + released []string }{ {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, + {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, + {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, } { t.Run(tc.name, func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), @@ -156,17 +202,29 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) } } - if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil { + end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + if tc.passed { + end.Outcome = runner.PipelineEndedOutcomePassed + } + if err := ix.HeardPins(ctx, nil, &end); err != nil { t.Fatal(err) } ix.JudgePins(ctx) + for _, s := range tc.then { + keep(s, nil) + } if got := kept(); !slices.Equal(got, tc.afterTheEnd) { t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd) } if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 { for _, k := range keys { - if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 { - t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v) + want := []string{} + if slices.Contains(tc.released, k) { + want = []string{ReleaseRecord("main", "reader")} + } + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) { + t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want) } } } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 67592964..d1a5703a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -170,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked", - "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index d71be480..5300ac2c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -89,6 +89,7 @@ "mesh-pipeline-runner.run-started", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.pipeline-ended", + "mesh-wants.pipeline-wanted", "node-engine.running", "operator-channel.decided", "npm-package-registry.published" -- 2.54.0 From 814244fd5a6564c9492269819dc69f4d837de2cd Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:58:45 +0100 Subject: [PATCH 2/3] mesh-delivery-indexer: a release pin only for a released mover, freed only by a newer release of it A passed pipeline's movers include ones the assigner never takes (superseded, no version released, a tier-late pipeline's later tiers); freeing their older release on the newer pipeline-wanted would remove the revision their want still names, and a mover releasing nothing would pin a revision per pipeline for good. The release record is now written only for a mover that released a version in the pipeline, and pipeline-wanted frees a module's older record only where the newer pipeline's own revision holds that module's release record. Tests: a pins table row (a passed mover with no version pins nothing), the freeing sequence (a passed pipeline releasing no version frees no older release, the next one that releases does), and TestIndexerWhole now says a pipeline-wanted on the bus after main.12's passed end and waits for main.11's release to be freed and main.11 removed, so the indexer's hearing of pipeline-wanted is tested end to end (review round 1). No test added. Builds: design 47 H-A-173 (hq design/47-H1-answers-59 21297155), H-A-170, H-A-168 --- .../takein_integration_test.go | 22 ++++++-- .../internal/index/pins.go | 30 +++++++++-- .../internal/index/pins_test.go | 50 ++++++++++++------- 3 files changed, 76 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 22c71590..cfcc1b8f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -340,7 +341,8 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende return } raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source, - Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}}) + Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1, + Version: "0.1." + s}}}) if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil { t.Fatal(err) } @@ -371,8 +373,9 @@ func absent(v []byte) bool { return v == nil } // - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // (H-A-114, H-A-118); nothing takes it in on a -// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its -// room and takes main.13 in on it; +// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the +// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and +// main.11 goes (H-A-168, H-A-170, H-A-173); // - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in // the hand-act log before it is asked, and take-in.taken says it skipped; // - a branch watched later is bound on registration-changed and its first revision taken in. @@ -506,6 +509,19 @@ func TestIndexerWhole(t *testing.T) { // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), + // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). + sayPipeline(t, ctx, js, true, "main", 12) + untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + var p index.PinValue + return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) + }) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { + t.Fatal(err) + } + untilKey(t, ctx, pins, index.PinKey("main", 11), absent) + untilRevisionGone(t, ctx, js, "main.11") if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 9b8117bc..e9111d80 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -85,9 +85,13 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) case ended != nil && ended.Revision > 0 && ended.Pipeline != "": // A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed, - // one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none. + // one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run + // on request's pipeline, which delivers nothing, pin none. if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" { for _, m := range ended.Movers { + if m.Version == "" { + continue + } if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil { return err } @@ -109,9 +113,11 @@ func ReleaseRecord(branch, module string) string { } // HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end, -// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names -// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A -// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing. +// tier-late included (design 47 H §751, H-A-168, H-A-170, H-A-173): a module whose release record the pipeline's own kept +// revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from +// every revision of the branch below it. A mover the revision holds no release of (superseded, no version released, a +// tier-late later tier) keeps its older release's revision. A pipeline whose revision is not kept frees nothing and says so; +// one applied twice changes nothing. func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) @@ -125,6 +131,20 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc if err != nil { return fmt.Errorf("%s: %w", name, err) } + own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision)) + if err != nil { + return err + } + var held PinValue + if found { + _ = json.Unmarshal(own, &held) + } + var released []meshcatalog.TakeInEntry + for _, m := range rev.Movers { + if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) { + released = append(released, m) + } + } pinKeys, err := ix.Pins.Keys(ctx) if err != nil { return err @@ -134,7 +154,7 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc if !ok || b != branch || r >= rev.Revision { continue } - for _, m := range rev.Movers { + for _, m := range released { v, _, found, err := ix.Pins.Get(ctx, k) if err != nil { return err diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index b2e7efa6..24ac07f4 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -92,8 +92,9 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v after the pin drop, want no main.1", keys) } - // H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's - // pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes. + // H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose + // mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees + // the older release on its pipeline-wanted (its wants written to the end), and the revision goes. mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} take := func(source int64) { t.Helper() @@ -103,32 +104,39 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Fatal(err) } } - end := func(source int64, outcome string) { + end := func(source int64, outcome, version string) { t.Helper() e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, - Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} if err := ix.HeardPins(ctx, nil, &e); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - take(10) - end(10, runner.PipelineEndedOutcomePassed) - take(11) - end(11, runner.PipelineEndedOutcomeFailed) - want("a passed release past a later failed pipeline", "main.10", "main.11") - take(12) - end(12, runner.PipelineEndedOutcomePassed) - want("two releases, the one between them gone", "main.10", "main.12") - for range 2 { // heard twice: changes nothing more - if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil { + wanted := func(source int64) { + t.Helper() + if err := ix.HeardPipelineWanted(ctx, "main", source); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - want("the newer release written everywhere", "main.12") - if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { - t.Errorf("the pin of main.12 holds %s, want its release alone", v) + take(10) + end(10, runner.PipelineEndedOutcomePassed, "0.1.10") + take(11) + end(11, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.10", "main.11") + take(12) + end(12, runner.PipelineEndedOutcomePassed, "") + wanted(12) + want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") + take(13) + end(13, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.10", "main.13") + wanted(13) + wanted(13) // heard twice: changes nothing more + want("the newer release written everywhere", "main.13") + if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.13 holds %s, want its release alone", v) } } @@ -158,6 +166,7 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { afterTheEnd []string base int64 // the base main.5 names (H-A-154) passed bool // main.5's pipeline passed: its release pins main.5 + unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) then []int64 // sources taken in with no mover after the end released []string }{ @@ -166,6 +175,8 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, } { @@ -203,7 +214,10 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { } } end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, - Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}} + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} + if tc.unreleased { + end.Movers[0].Version = "" + } if tc.passed { end.Outcome = runner.PipelineEndedOutcomePassed } -- 2.54.0 From bf24bb97408fe8cad0169b9637bbdce7638f94b2 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 18:01:56 +0100 Subject: [PATCH 3/3] mesh-delivery-indexer: a tier-late pipeline-wanted frees no release A tier-late pipeline's later-tier movers released a version, so the newer revision holds their release record, yet their wants were not written: freeing their older release on the tier-late event would remove the revision their want names. HeardPipelineWanted now reads the event's outcome: tier-late frees nothing, and the same pipeline's later pipeline-wanted with outcome running frees as before. The pins test's freeing sequence says a tier-late event first (nothing freed), then the running one (the older release freed); the events test's row hands the outcome on. No test added. Builds: design 47 H-A-174 (hq design/47-H1-answers-59 84e2514d), H-A-173 --- .../cmd/mesh-delivery-indexer/named_test.go | 8 ++++---- .../cmd/mesh-delivery-indexer/runs.go | 4 ++-- .../internal/index/pins.go | 20 ++++++++++++------- .../internal/index/pins_test.go | 16 +++++++++------ 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 7fafd036..ce7458a1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 switch { case row.wanted: heardPipelineWanted(context.Background(), ix, m, t.Logf) - if ix.wanted != "main.12" { - t.Errorf("%s: freed the releases below %q, want main.12", row.name, ix.wanted) + if ix.wanted != "main.12 tier-late" { + t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted) } case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) @@ -164,8 +164,8 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } -func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64) error { - f.wanted = fmt.Sprintf("%s.%d", branch, source) +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error { + f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome) return f.err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index 033be1c4..fdbf67aa 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -21,7 +21,7 @@ type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error - HeardPipelineWanted(ctx context.Context, branch string, source int64) error + HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error JudgePins(ctx context.Context) } @@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun _ = m.Term() return } - err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source) + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome) settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) if err == nil { ix.JudgePins(ctx) diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index e9111d80..33816fa8 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -14,6 +14,7 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -112,16 +113,21 @@ func ReleaseRecord(branch, module string) string { return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module) } -// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end, -// tier-late included (design 47 H §751, H-A-168, H-A-170, H-A-173): a module whose release record the pipeline's own kept -// revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from -// every revision of the branch below it. A mover the revision holds no release of (superseded, no version released, a -// tier-late later tier) keeps its older release's revision. A pipeline whose revision is not kept frees nothing and says so; -// one applied twice changes nothing. -func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error { +// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said when a pipeline's wants are written to the end +// (design 47 H §751, H-A-168, H-A-170, H-A-173, H-A-174). Outcome running: a module whose release record the pipeline's own +// kept revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from +// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its +// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release +// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and +// says so; one applied twice changes nothing. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) } + if outcome != meshwants.PipelineWantedOutcomeRunning { + ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome) + return nil + } name, err := revisionOfSource(ctx, ix.Revisions, branch, source) if err != nil { ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 24ac07f4..4a255ebd 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its @@ -94,7 +95,8 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } // H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose // mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees - // the older release on its pipeline-wanted (its wants written to the end), and the revision goes. + // the older release on its pipeline-wanted running (its wants written to the end), not on a tier-late one (H-A-174), and the + // revision goes. mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} take := func(source int64) { t.Helper() @@ -113,9 +115,9 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } ix.JudgePins(ctx) } - wanted := func(source int64) { + wanted := func(source int64, outcome string) { t.Helper() - if err := ix.HeardPipelineWanted(ctx, "main", source); err != nil { + if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil { t.Fatal(err) } ix.JudgePins(ctx) @@ -127,13 +129,15 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te want("a passed release past a later failed pipeline", "main.10", "main.11") take(12) end(12, runner.PipelineEndedOutcomePassed, "") - wanted(12) + wanted(12, meshwants.PipelineWantedOutcomeRunning) want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") take(13) end(13, runner.PipelineEndedOutcomePassed, "0.1.13") want("two releases, the unreleased one between them gone", "main.10", "main.13") - wanted(13) - wanted(13) // heard twice: changes nothing more + wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing + want("a tier-late pipeline-wanted", "main.10", "main.13") + wanted(13, meshwants.PipelineWantedOutcomeRunning) + wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more want("the newer release written everywhere", "main.13") if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { t.Errorf("the pin of main.13 holds %s, want its release alone", v) -- 2.54.0