Design 47 H1: a release pins the revision its entry lives in; freed by the module's next release (H-A-168, H-A-170, H-A-173, H-A-174) #280
@@ -303,8 +303,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
// The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register
|
||||
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed
|
||||
// and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them.
|
||||
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed,
|
||||
// the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted
|
||||
// (H-A-170), as the grants part makes them.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -313,7 +314,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
}
|
||||
for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged,
|
||||
meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll,
|
||||
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} {
|
||||
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
// or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked.
|
||||
for _, row := range []struct {
|
||||
name, body string
|
||||
ended bool
|
||||
ended, wanted bool
|
||||
err error
|
||||
acked, termd int
|
||||
started, endedOf string
|
||||
}{
|
||||
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
|
||||
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
|
||||
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
|
||||
{name: "not json", body: `nope`, termd: 1},
|
||||
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
|
||||
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
|
||||
} {
|
||||
ix := &fakeRuns{err: row.err}
|
||||
m := &fakeMsg{subject: "x", data: []byte(row.body)}
|
||||
if row.ended {
|
||||
switch {
|
||||
case row.wanted:
|
||||
heardPipelineWanted(context.Background(), ix, m, t.Logf)
|
||||
if ix.wanted != "main.12 tier-late" {
|
||||
t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted)
|
||||
}
|
||||
case row.ended:
|
||||
heardPipelineEnded(context.Background(), ix, m, t.Logf)
|
||||
} else {
|
||||
default:
|
||||
heardRunStarted(context.Background(), ix, m, t.Logf)
|
||||
}
|
||||
if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf {
|
||||
@@ -139,8 +146,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
|
||||
// fakeRuns records the run events handed to it and answers err.
|
||||
type fakeRuns struct {
|
||||
err error
|
||||
started, ended string
|
||||
err error
|
||||
started, ended, wanted string
|
||||
}
|
||||
|
||||
func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error {
|
||||
@@ -157,4 +164,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error {
|
||||
f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeRuns) JudgePins(context.Context) {}
|
||||
|
||||
@@ -6,15 +6,17 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on
|
||||
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins,
|
||||
// and only then acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release
|
||||
// pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable
|
||||
// consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then
|
||||
// acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||
|
||||
// pinsStreams are the streams of the runner's events the indexer's pins follow.
|
||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded}
|
||||
// pinsStreams are the streams of the events the indexer's pins follow.
|
||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted}
|
||||
|
||||
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
|
||||
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
|
||||
|
||||
@@ -8,17 +8,20 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
|
||||
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins).
|
||||
// and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted,
|
||||
// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins).
|
||||
type runTaker interface {
|
||||
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
|
||||
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
|
||||
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
|
||||
HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error
|
||||
JudgePins(ctx context.Context)
|
||||
}
|
||||
|
||||
@@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
||||
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
|
||||
{meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }},
|
||||
} {
|
||||
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
|
||||
cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf)
|
||||
@@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
|
||||
// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late
|
||||
// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168,
|
||||
// H-A-170).
|
||||
func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
|
||||
var e meshwants.PipelineWanted
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 {
|
||||
logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome)
|
||||
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
|
||||
if err == nil {
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
|
||||
switch {
|
||||
case err == nil:
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -339,17 +341,22 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source,
|
||||
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}})
|
||||
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1,
|
||||
Version: "0.1." + s}}})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// pinned is whether a pins value names the record.
|
||||
func pinned(record string) func([]byte) bool {
|
||||
// pinned is whether a pins value names exactly the records, in any order.
|
||||
func pinned(records ...string) func([]byte) bool {
|
||||
return func(v []byte) bool {
|
||||
var p index.PinValue
|
||||
return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record
|
||||
if v == nil || json.Unmarshal(v, &p) != nil {
|
||||
return false
|
||||
}
|
||||
slices.Sort(p.Records)
|
||||
return slices.Equal(p.Records, slices.Sorted(slices.Values(records)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,8 +373,9 @@ func absent(v []byte) bool { return v == nil }
|
||||
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
|
||||
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
|
||||
// (H-A-114, H-A-118); nothing takes it in on a
|
||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
|
||||
// room and takes main.13 in on it;
|
||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
|
||||
// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and
|
||||
// main.11 goes (H-A-168, H-A-170, H-A-173);
|
||||
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
|
||||
// the hand-act log before it is asked, and take-in.taken says it skipped;
|
||||
// - a branch watched later is bound on registration-changed and its first revision taken in.
|
||||
@@ -484,7 +492,9 @@ func TestIndexerWhole(t *testing.T) {
|
||||
return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() &&
|
||||
hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13
|
||||
})
|
||||
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held.
|
||||
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the
|
||||
// write, so it is waited for before the consumer is read.
|
||||
heard.acked(t, "main")
|
||||
if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 {
|
||||
@@ -496,8 +506,22 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
|
||||
}
|
||||
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
|
||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
|
||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
|
||||
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
|
||||
sayPipeline(t, ctx, js, true, "main", 12)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
|
||||
var p index.PinValue
|
||||
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
|
||||
})
|
||||
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
|
||||
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
|
||||
untilRevisionGone(t, ctx, js, "main.11")
|
||||
if n := len(heard.pending); n != 0 {
|
||||
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
|
||||
}
|
||||
|
||||
@@ -14,14 +14,15 @@ import (
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
|
||||
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each
|
||||
// running pipeline's (design 47 H §743, §749).
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each
|
||||
// running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170).
|
||||
const (
|
||||
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
|
||||
PinsState = "pins"
|
||||
@@ -84,6 +85,19 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
||||
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
||||
// A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed,
|
||||
// one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run
|
||||
// on request's pipeline, which delivers nothing, pin none.
|
||||
if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" {
|
||||
for _, m := range ended.Movers {
|
||||
if m.Version == "" {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
record := PipelineRecord(ended.Pipeline)
|
||||
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
|
||||
return err
|
||||
@@ -93,6 +107,77 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release.<branch>.<module>`,
|
||||
// each name escaped as a key's name is (design 47 H-A-168, H-A-170).
|
||||
func ReleaseRecord(branch, module string) string {
|
||||
return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module)
|
||||
}
|
||||
|
||||
// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said when a pipeline's wants are written to the end
|
||||
// (design 47 H §751, H-A-168, H-A-170, H-A-173, H-A-174). Outcome running: a module whose release record the pipeline's own
|
||||
// kept revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from
|
||||
// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its
|
||||
// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release
|
||||
// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and
|
||||
// says so; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error {
|
||||
if ix.Pins == nil || ix.Revisions == nil {
|
||||
return errors.New("the indexer keeps no state " + PinsState)
|
||||
}
|
||||
if outcome != meshwants.PipelineWantedOutcomeRunning {
|
||||
ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome)
|
||||
return nil
|
||||
}
|
||||
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
|
||||
if err != nil {
|
||||
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
|
||||
return nil
|
||||
}
|
||||
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var held PinValue
|
||||
if found {
|
||||
_ = json.Unmarshal(own, &held)
|
||||
}
|
||||
var released []meshcatalog.TakeInEntry
|
||||
for _, m := range rev.Movers {
|
||||
if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) {
|
||||
released = append(released, m)
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r >= rev.Revision {
|
||||
continue
|
||||
}
|
||||
for _, m := range released {
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
record := ReleaseRecord(branch, m.Module)
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
|
||||
// same pipeline (H-A-147), which the pipeline-ended does not name.
|
||||
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
@@ -92,6 +93,55 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
|
||||
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
|
||||
t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
|
||||
}
|
||||
// H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose
|
||||
// mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees
|
||||
// the older release on its pipeline-wanted running (its wants written to the end), not on a tier-late one (H-A-174), and the
|
||||
// revision goes.
|
||||
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
|
||||
take := func(source int64) {
|
||||
t.Helper()
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("r%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
end := func(source int64, outcome, version string) {
|
||||
t.Helper()
|
||||
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
|
||||
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}}
|
||||
if err := ix.HeardPins(ctx, nil, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
wanted := func(source int64, outcome string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
take(10)
|
||||
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
|
||||
take(11)
|
||||
end(11, runner.PipelineEndedOutcomeFailed, "")
|
||||
want("a passed release past a later failed pipeline", "main.10", "main.11")
|
||||
take(12)
|
||||
end(12, runner.PipelineEndedOutcomePassed, "")
|
||||
wanted(12, meshwants.PipelineWantedOutcomeRunning)
|
||||
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
|
||||
take(13)
|
||||
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
|
||||
want("two releases, the unreleased one between them gone", "main.10", "main.13")
|
||||
wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
|
||||
want("a tier-late pipeline-wanted", "main.10", "main.13")
|
||||
wanted(13, meshwants.PipelineWantedOutcomeRunning)
|
||||
wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
|
||||
want("the newer release written everywhere", "main.13")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
|
||||
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRecords(t *testing.T, v []byte) []string {
|
||||
@@ -107,7 +157,8 @@ func mustRecords(t *testing.T, v []byte) []string {
|
||||
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
|
||||
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
|
||||
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
|
||||
// its own and names no base; the pipeline's end frees both.
|
||||
// its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release
|
||||
// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current.
|
||||
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
@@ -117,11 +168,21 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
before []int64 // sources taken in with no mover before the take-in of source 5
|
||||
whileRuns []string
|
||||
afterTheEnd []string
|
||||
base int64 // the base main.5 names (H-A-154)
|
||||
base int64 // the base main.5 names (H-A-154)
|
||||
passed bool // main.5's pipeline passed: its release pins main.5
|
||||
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
|
||||
then []int64 // sources taken in with no mover after the end
|
||||
released []string
|
||||
}{
|
||||
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
|
||||
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
|
||||
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
|
||||
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
|
||||
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
|
||||
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
@@ -156,17 +217,32 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
|
||||
}
|
||||
}
|
||||
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
|
||||
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
|
||||
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
|
||||
if tc.unreleased {
|
||||
end.Movers[0].Version = ""
|
||||
}
|
||||
if tc.passed {
|
||||
end.Outcome = runner.PipelineEndedOutcomePassed
|
||||
}
|
||||
if err := ix.HeardPins(ctx, nil, &end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
for _, s := range tc.then {
|
||||
keep(s, nil)
|
||||
}
|
||||
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
|
||||
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
|
||||
}
|
||||
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
|
||||
for _, k := range keys {
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
|
||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
|
||||
want := []string{}
|
||||
if slices.Contains(tc.released, k) {
|
||||
want = []string{ReleaseRecord("main", "reader")}
|
||||
}
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) {
|
||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
|
||||
@@ -89,6 +89,7 @@
|
||||
"mesh-pipeline-runner.run-started",
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.pipeline-ended",
|
||||
"mesh-wants.pipeline-wanted",
|
||||
"node-engine.running",
|
||||
"operator-channel.decided",
|
||||
"npm-package-registry.published"
|
||||
|
||||
Reference in New Issue
Block a user