diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 88112068..745cfab5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -303,8 +303,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) // The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register - // (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed - // and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them. + // (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed, + // the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted + // (H-A-170), as the grants part makes them. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil { t.Fatal(err) } @@ -313,7 +314,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged, meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll, - meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} { + meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index c3d723f4..c57388a2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 // or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked. for _, row := range []struct { name, body string - ended bool + ended, wanted bool err error acked, termd int started, endedOf string }{ {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, + {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1}, {name: "not json", body: `nope`, termd: 1}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, } { ix := &fakeRuns{err: row.err} m := &fakeMsg{subject: "x", data: []byte(row.body)} - if row.ended { + switch { + case row.wanted: + heardPipelineWanted(context.Background(), ix, m, t.Logf) + if ix.wanted != "main.12 tier-late" { + t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted) + } + case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) - } else { + default: heardRunStarted(context.Background(), ix, m, t.Logf) } if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf { @@ -144,8 +151,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { // fakeRuns records the run events handed to it and answers err. type fakeRuns struct { - err error - started, ended string + err error + started, ended, wanted string } func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error { @@ -162,4 +169,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error { + f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome) + return f.err +} + func (f *fakeRuns) JudgePins(context.Context) {} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go index 49f9132f..a5104616 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go @@ -6,15 +6,17 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" ) -// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on -// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, -// and only then acknowledged, so one said while the indexer was down is applied when it comes back. +// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release +// pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable +// consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then +// acknowledged, so one said while the indexer was down is applied when it comes back. -// pinsStreams are the streams of the runner's events the indexer's pins follow. -var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded} +// pinsStreams are the streams of the events the indexer's pins follow. +var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted} // pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet. func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index 9a3d42b5..fdbf67aa 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -8,17 +8,20 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) // runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline), -// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins). +// and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted, +// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins). type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error + HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error JudgePins(ctx context.Context) } @@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, {meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }}, + {meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }}, } { _ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error { cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf) @@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func ix.JudgePins(ctx) } +// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late +// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168, +// H-A-170). +func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { + var e meshwants.PipelineWanted + if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 { + logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome) + settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) + if err == nil { + ix.JudgePins(ctx) + } +} + func settleRun(m delivered, err error, what string, logf func(string, ...any)) { switch { case err == nil: diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 98cc851a..cfcc1b8f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "os" + "slices" "strconv" "strings" "sync" @@ -20,6 +21,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -339,17 +341,22 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende return } raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source, - Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}}) + Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1, + Version: "0.1." + s}}}) if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil { t.Fatal(err) } } -// pinned is whether a pins value names the record. -func pinned(record string) func([]byte) bool { +// pinned is whether a pins value names exactly the records, in any order. +func pinned(records ...string) func([]byte) bool { return func(v []byte) bool { var p index.PinValue - return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record + if v == nil || json.Unmarshal(v, &p) != nil { + return false + } + slices.Sort(p.Records) + return slices.Equal(p.Records, slices.Sorted(slices.Values(records))) } } @@ -366,8 +373,9 @@ func absent(v []byte) bool { return v == nil } // - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // (H-A-114, H-A-118); nothing takes it in on a -// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its -// room and takes main.13 in on it; +// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the +// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and +// main.11 goes (H-A-168, H-A-170, H-A-173); // - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in // the hand-act log before it is asked, and take-in.taken says it skipped; // - a branch watched later is bound on registration-changed and its first revision taken in. @@ -484,7 +492,9 @@ func TestIndexerWhole(t *testing.T) { return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() && hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13 }) - // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held. + // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the + // write, so it is waited for before the consumer is read. + heard.acked(t, "main") if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil { t.Fatal(err) } else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 { @@ -496,8 +506,22 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147) - untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). + untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) + untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), + // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). + sayPipeline(t, ctx, js, true, "main", 12) + untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + var p index.PinValue + return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) + }) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { + t.Fatal(err) + } + untilKey(t, ctx, pins, index.PinKey("main", 11), absent) + untilRevisionGone(t, ctx, js, "main.11") if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 5fa6c844..33816fa8 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -14,14 +14,15 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" ) // The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153, // §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `.` (the watched branch -// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each -// running pipeline's (design 47 H §743, §749). +// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each +// running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170). const ( // PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB). PinsState = "pins" @@ -84,6 +85,19 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte case started != nil && started.Revision > 0 && started.Pipeline != "": return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) case ended != nil && ended.Revision > 0 && ended.Pipeline != "": + // A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed, + // one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run + // on request's pipeline, which delivers nothing, pin none. + if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" { + for _, m := range ended.Movers { + if m.Version == "" { + continue + } + if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil { + return err + } + } + } record := PipelineRecord(ended.Pipeline) if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil { return err @@ -93,6 +107,77 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte return nil } +// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release..`, +// each name escaped as a key's name is (design 47 H-A-168, H-A-170). +func ReleaseRecord(branch, module string) string { + return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module) +} + +// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said when a pipeline's wants are written to the end +// (design 47 H §751, H-A-168, H-A-170, H-A-173, H-A-174). Outcome running: a module whose release record the pipeline's own +// kept revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from +// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its +// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release +// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and +// says so; one applied twice changes nothing. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error { + if ix.Pins == nil || ix.Revisions == nil { + return errors.New("the indexer keeps no state " + PinsState) + } + if outcome != meshwants.PipelineWantedOutcomeRunning { + ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome) + return nil + } + name, err := revisionOfSource(ctx, ix.Revisions, branch, source) + if err != nil { + ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) + return nil + } + rev, _, err := ReadRevision(ctx, ix.Revisions, name) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision)) + if err != nil { + return err + } + var held PinValue + if found { + _ = json.Unmarshal(own, &held) + } + var released []meshcatalog.TakeInEntry + for _, m := range rev.Movers { + if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) { + released = append(released, m) + } + } + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return err + } + for _, k := range pinKeys { + b, r, ok := ParsePinKey(k) + if !ok || b != branch || r >= rev.Revision { + continue + } + for _, m := range released { + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return err + } + var pv PinValue + record := ReleaseRecord(branch, m.Module) + if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) { + continue + } + if err := ix.unpin(ctx, k, record); err != nil { + return err + } + } + } + return nil +} + // unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the // same pipeline (H-A-147), which the pipeline-ended does not name. func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index a2f89cc6..4a255ebd 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its @@ -92,6 +93,55 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v after the pin drop, want no main.1", keys) } + // H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose + // mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees + // the older release on its pipeline-wanted running (its wants written to the end), not on a tier-late one (H-A-174), and the + // revision goes. + mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}} + take := func(source int64) { + t.Helper() + e := reader + e.Commit = fmt.Sprintf("r%d", source) + if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil { + t.Fatal(err) + } + } + end := func(source int64, outcome, version string) { + t.Helper() + e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} + if err := ix.HeardPins(ctx, nil, &e); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + } + wanted := func(source int64, outcome string) { + t.Helper() + if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + } + take(10) + end(10, runner.PipelineEndedOutcomePassed, "0.1.10") + take(11) + end(11, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.10", "main.11") + take(12) + end(12, runner.PipelineEndedOutcomePassed, "") + wanted(12, meshwants.PipelineWantedOutcomeRunning) + want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") + take(13) + end(13, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.10", "main.13") + wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing + want("a tier-late pipeline-wanted", "main.10", "main.13") + wanted(13, meshwants.PipelineWantedOutcomeRunning) + wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more + want("the newer release written everywhere", "main.13") + if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.13 holds %s, want its release alone", v) + } } func mustRecords(t *testing.T, v []byte) []string { @@ -107,7 +157,8 @@ func mustRecords(t *testing.T, v []byte) []string { // H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept // before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline // runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only -// its own and names no base; the pipeline's end frees both. +// its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release +// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current. func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { ctx := context.Background() reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} @@ -117,11 +168,21 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string - base int64 // the base main.5 names (H-A-154) + base int64 // the base main.5 names (H-A-154) + passed bool // main.5's pipeline passed: its release pins main.5 + unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) + then []int64 // sources taken in with no mover after the end + released []string }{ {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, + {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, + {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, + base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, } { t.Run(tc.name, func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), @@ -156,17 +217,32 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) } } - if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil { + end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, + Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} + if tc.unreleased { + end.Movers[0].Version = "" + } + if tc.passed { + end.Outcome = runner.PipelineEndedOutcomePassed + } + if err := ix.HeardPins(ctx, nil, &end); err != nil { t.Fatal(err) } ix.JudgePins(ctx) + for _, s := range tc.then { + keep(s, nil) + } if got := kept(); !slices.Equal(got, tc.afterTheEnd) { t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd) } if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 { for _, k := range keys { - if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 { - t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v) + want := []string{} + if slices.Contains(tc.released, k) { + want = []string{ReleaseRecord("main", "reader")} + } + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) { + t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want) } } } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 67592964..d1a5703a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -170,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked", - "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index d71be480..5300ac2c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -89,6 +89,7 @@ "mesh-pipeline-runner.run-started", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.pipeline-ended", + "mesh-wants.pipeline-wanted", "node-engine.running", "operator-channel.decided", "npm-package-registry.published"