mesh-delivery-indexer: a passed pipeline's release pins its revision until a newer one is the want
The pin judge removed main.23 and its entries when main.24 became current, while the assigner had just written main.23's builder as the want: the node-engine asked its entry and was told none is kept, holding the want for good. A passed pipeline-ended now pins the pipeline's kept revision under a record release.<branch>.<module>, one per mover, beside the pipeline's own pin; the seat mesh-wants' pipeline-wanted of a newer pipeline on that branch (its wants written to the end, tier-late included) frees that record from every revision below the newer one, for each module the newer revision's movers name. The indexer consumes pipeline-wanted (its manifest says so). A revision a release pin holds keeps its entries, as removeUnlisted already does for any kept revision. Tests: rows in the pins table (a passed pipeline's revision kept once no longer current; a failed one's goes), the freeing in the pins test (a release past a failed pipeline, freed by the next passed one's pipeline-wanted, twice changing nothing), and a pipeline-wanted row in the events test. TestIndexerWhole now reads main.11's release beside its validate base, and waits for the held source-moved's acknowledgement before it reads the consumer (the write comes first). No test added. Builds: design 47 H-A-168, H-A-170 (hq design/47-H1-answers-59 87b43ec8), H-A-147, H §749
This commit is contained in:
@@ -303,8 +303,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
t.Cleanup(cancel)
|
t.Cleanup(cancel)
|
||||||
// The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register
|
// The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register
|
||||||
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed
|
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed,
|
||||||
// and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them.
|
// the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted
|
||||||
|
// (H-A-170), as the grants part makes them.
|
||||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil {
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -313,7 +314,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
|||||||
}
|
}
|
||||||
for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged,
|
for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged,
|
||||||
meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll,
|
meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll,
|
||||||
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} {
|
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} {
|
||||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil {
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
|||||||
// or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked.
|
// or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked.
|
||||||
for _, row := range []struct {
|
for _, row := range []struct {
|
||||||
name, body string
|
name, body string
|
||||||
ended bool
|
ended, wanted bool
|
||||||
err error
|
err error
|
||||||
acked, termd int
|
acked, termd int
|
||||||
started, endedOf string
|
started, endedOf string
|
||||||
}{
|
}{
|
||||||
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
|
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
|
||||||
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
|
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
|
||||||
|
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
|
||||||
{name: "not json", body: `nope`, termd: 1},
|
{name: "not json", body: `nope`, termd: 1},
|
||||||
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
|
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
|
||||||
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
|
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
|
||||||
} {
|
} {
|
||||||
ix := &fakeRuns{err: row.err}
|
ix := &fakeRuns{err: row.err}
|
||||||
m := &fakeMsg{subject: "x", data: []byte(row.body)}
|
m := &fakeMsg{subject: "x", data: []byte(row.body)}
|
||||||
if row.ended {
|
switch {
|
||||||
|
case row.wanted:
|
||||||
|
heardPipelineWanted(context.Background(), ix, m, t.Logf)
|
||||||
|
if ix.wanted != "main.12" {
|
||||||
|
t.Errorf("%s: freed the releases below %q, want main.12", row.name, ix.wanted)
|
||||||
|
}
|
||||||
|
case row.ended:
|
||||||
heardPipelineEnded(context.Background(), ix, m, t.Logf)
|
heardPipelineEnded(context.Background(), ix, m, t.Logf)
|
||||||
} else {
|
default:
|
||||||
heardRunStarted(context.Background(), ix, m, t.Logf)
|
heardRunStarted(context.Background(), ix, m, t.Logf)
|
||||||
}
|
}
|
||||||
if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf {
|
if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf {
|
||||||
@@ -140,7 +147,7 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
|||||||
// fakeRuns records the run events handed to it and answers err.
|
// fakeRuns records the run events handed to it and answers err.
|
||||||
type fakeRuns struct {
|
type fakeRuns struct {
|
||||||
err error
|
err error
|
||||||
started, ended string
|
started, ended, wanted string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error {
|
func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error {
|
||||||
@@ -157,4 +164,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64) error {
|
||||||
|
f.wanted = fmt.Sprintf("%s.%d", branch, source)
|
||||||
|
return f.err
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeRuns) JudgePins(context.Context) {}
|
func (f *fakeRuns) JudgePins(context.Context) {}
|
||||||
|
|||||||
@@ -6,15 +6,17 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
"git.novox.be/novox/mesh-sdk/go/process"
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on
|
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release
|
||||||
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins,
|
// pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable
|
||||||
// and only then acknowledged, so one said while the indexer was down is applied when it comes back.
|
// consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then
|
||||||
|
// acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||||
|
|
||||||
// pinsStreams are the streams of the runner's events the indexer's pins follow.
|
// pinsStreams are the streams of the events the indexer's pins follow.
|
||||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded}
|
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted}
|
||||||
|
|
||||||
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
|
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
|
||||||
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
|
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
|
||||||
|
|||||||
@@ -8,17 +8,20 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
"git.novox.be/novox/mesh-sdk/go/process"
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
|
||||||
"mesh-delivery-indexer/internal/index"
|
"mesh-delivery-indexer/internal/index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
|
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
|
||||||
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins).
|
// and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted,
|
||||||
|
// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins).
|
||||||
type runTaker interface {
|
type runTaker interface {
|
||||||
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
|
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
|
||||||
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
|
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
|
||||||
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
|
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
|
||||||
|
HeardPipelineWanted(ctx context.Context, branch string, source int64) error
|
||||||
JudgePins(ctx context.Context)
|
JudgePins(ctx context.Context)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
|||||||
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
|
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
|
||||||
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
|
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
|
||||||
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
|
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
|
||||||
|
{meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }},
|
||||||
} {
|
} {
|
||||||
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
|
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
|
||||||
cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf)
|
cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf)
|
||||||
@@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
|
|||||||
ix.JudgePins(ctx)
|
ix.JudgePins(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late
|
||||||
|
// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168,
|
||||||
|
// H-A-170).
|
||||||
|
func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
|
||||||
|
var e meshwants.PipelineWanted
|
||||||
|
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 {
|
||||||
|
logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err)
|
||||||
|
_ = m.Term()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source)
|
||||||
|
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
|
||||||
|
if err == nil {
|
||||||
|
ix.JudgePins(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
|
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
|
||||||
switch {
|
switch {
|
||||||
case err == nil:
|
case err == nil:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -345,11 +346,15 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pinned is whether a pins value names the record.
|
// pinned is whether a pins value names exactly the records, in any order.
|
||||||
func pinned(record string) func([]byte) bool {
|
func pinned(records ...string) func([]byte) bool {
|
||||||
return func(v []byte) bool {
|
return func(v []byte) bool {
|
||||||
var p index.PinValue
|
var p index.PinValue
|
||||||
return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record
|
if v == nil || json.Unmarshal(v, &p) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
slices.Sort(p.Records)
|
||||||
|
return slices.Equal(p.Records, slices.Sorted(slices.Values(records)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -484,7 +489,9 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() &&
|
return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() &&
|
||||||
hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13
|
hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13
|
||||||
})
|
})
|
||||||
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held.
|
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the
|
||||||
|
// write, so it is waited for before the consumer is read.
|
||||||
|
heard.acked(t, "main")
|
||||||
if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil {
|
if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
} else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 {
|
} else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 {
|
||||||
@@ -496,7 +503,8 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
|
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
|
||||||
}
|
}
|
||||||
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
|
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
|
||||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
|
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
|
||||||
|
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
|
||||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||||
if n := len(heard.pending); n != 0 {
|
if n := len(heard.pending); n != 0 {
|
||||||
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
|
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ import (
|
|||||||
|
|
||||||
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
|
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
|
||||||
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
|
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
|
||||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each
|
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each
|
||||||
// running pipeline's (design 47 H §743, §749).
|
// running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170).
|
||||||
const (
|
const (
|
||||||
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
|
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
|
||||||
PinsState = "pins"
|
PinsState = "pins"
|
||||||
@@ -84,6 +84,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
|||||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||||
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
||||||
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
||||||
|
// A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed,
|
||||||
|
// one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none.
|
||||||
|
if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" {
|
||||||
|
for _, m := range ended.Movers {
|
||||||
|
if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
record := PipelineRecord(ended.Pipeline)
|
record := PipelineRecord(ended.Pipeline)
|
||||||
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
|
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -93,6 +102,56 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release.<branch>.<module>`,
|
||||||
|
// each name escaped as a key's name is (design 47 H-A-168, H-A-170).
|
||||||
|
func ReleaseRecord(branch, module string) string {
|
||||||
|
return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end,
|
||||||
|
// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names
|
||||||
|
// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A
|
||||||
|
// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing.
|
||||||
|
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error {
|
||||||
|
if ix.Pins == nil || ix.Revisions == nil {
|
||||||
|
return errors.New("the indexer keeps no state " + PinsState)
|
||||||
|
}
|
||||||
|
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
|
||||||
|
if err != nil {
|
||||||
|
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", name, err)
|
||||||
|
}
|
||||||
|
pinKeys, err := ix.Pins.Keys(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, k := range pinKeys {
|
||||||
|
b, r, ok := ParsePinKey(k)
|
||||||
|
if !ok || b != branch || r >= rev.Revision {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range rev.Movers {
|
||||||
|
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var pv PinValue
|
||||||
|
record := ReleaseRecord(branch, m.Module)
|
||||||
|
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := ix.unpin(ctx, k, record); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
|
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
|
||||||
// same pipeline (H-A-147), which the pipeline-ended does not name.
|
// same pipeline (H-A-147), which the pipeline-ended does not name.
|
||||||
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||||
|
|||||||
@@ -92,6 +92,44 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
|
|||||||
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
|
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
|
||||||
t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
|
t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
|
||||||
}
|
}
|
||||||
|
// H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's
|
||||||
|
// pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes.
|
||||||
|
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
|
||||||
|
take := func(source int64) {
|
||||||
|
t.Helper()
|
||||||
|
e := reader
|
||||||
|
e.Commit = fmt.Sprintf("r%d", source)
|
||||||
|
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end := func(source int64, outcome string) {
|
||||||
|
t.Helper()
|
||||||
|
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
|
||||||
|
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
|
||||||
|
if err := ix.HeardPins(ctx, nil, &e); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ix.JudgePins(ctx)
|
||||||
|
}
|
||||||
|
take(10)
|
||||||
|
end(10, runner.PipelineEndedOutcomePassed)
|
||||||
|
take(11)
|
||||||
|
end(11, runner.PipelineEndedOutcomeFailed)
|
||||||
|
want("a passed release past a later failed pipeline", "main.10", "main.11")
|
||||||
|
take(12)
|
||||||
|
end(12, runner.PipelineEndedOutcomePassed)
|
||||||
|
want("two releases, the one between them gone", "main.10", "main.12")
|
||||||
|
for range 2 { // heard twice: changes nothing more
|
||||||
|
if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ix.JudgePins(ctx)
|
||||||
|
}
|
||||||
|
want("the newer release written everywhere", "main.12")
|
||||||
|
if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
|
||||||
|
t.Errorf("the pin of main.12 holds %s, want its release alone", v)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mustRecords(t *testing.T, v []byte) []string {
|
func mustRecords(t *testing.T, v []byte) []string {
|
||||||
@@ -107,7 +145,8 @@ func mustRecords(t *testing.T, v []byte) []string {
|
|||||||
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
|
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
|
||||||
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
|
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
|
||||||
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
|
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
|
||||||
// its own and names no base; the pipeline's end frees both.
|
// its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release
|
||||||
|
// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current.
|
||||||
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||||
@@ -118,10 +157,17 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
|||||||
whileRuns []string
|
whileRuns []string
|
||||||
afterTheEnd []string
|
afterTheEnd []string
|
||||||
base int64 // the base main.5 names (H-A-154)
|
base int64 // the base main.5 names (H-A-154)
|
||||||
|
passed bool // main.5's pipeline passed: its release pins main.5
|
||||||
|
then []int64 // sources taken in with no mover after the end
|
||||||
|
released []string
|
||||||
}{
|
}{
|
||||||
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
|
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
|
||||||
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
|
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
|
||||||
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
|
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
|
||||||
|
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||||
|
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
|
||||||
|
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||||
|
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||||
@@ -156,17 +202,29 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
|||||||
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
|
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
|
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
|
||||||
|
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
|
||||||
|
if tc.passed {
|
||||||
|
end.Outcome = runner.PipelineEndedOutcomePassed
|
||||||
|
}
|
||||||
|
if err := ix.HeardPins(ctx, nil, &end); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
ix.JudgePins(ctx)
|
ix.JudgePins(ctx)
|
||||||
|
for _, s := range tc.then {
|
||||||
|
keep(s, nil)
|
||||||
|
}
|
||||||
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
|
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
|
||||||
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
|
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
|
||||||
}
|
}
|
||||||
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
|
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
|
want := []string{}
|
||||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
|
if slices.Contains(tc.released, k) {
|
||||||
|
want = []string{ReleaseRecord("main", "reader")}
|
||||||
|
}
|
||||||
|
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) {
|
||||||
|
t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
|||||||
// internal/broker/state.go:62-93).
|
// internal/broker/state.go:62-93).
|
||||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
|
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
|
||||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||||
}
|
}
|
||||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||||
|
|||||||
@@ -89,6 +89,7 @@
|
|||||||
"mesh-pipeline-runner.run-started",
|
"mesh-pipeline-runner.run-started",
|
||||||
"mesh-pipeline-runner.pipeline-started",
|
"mesh-pipeline-runner.pipeline-started",
|
||||||
"mesh-pipeline-runner.pipeline-ended",
|
"mesh-pipeline-runner.pipeline-ended",
|
||||||
|
"mesh-wants.pipeline-wanted",
|
||||||
"node-engine.running",
|
"node-engine.running",
|
||||||
"operator-channel.decided",
|
"operator-channel.decided",
|
||||||
"npm-package-registry.published"
|
"npm-package-registry.published"
|
||||||
|
|||||||
Reference in New Issue
Block a user