mesh-delivery-indexer: a passed pipeline's release pins its revision until a newer one is the want

The pin judge removed main.23 and its entries when main.24 became current, while the assigner had
just written main.23's builder as the want: the node-engine asked its entry and was told none is
kept, holding the want for good. A passed pipeline-ended now pins the pipeline's kept revision
under a record release.<branch>.<module>, one per mover, beside the pipeline's own pin; the seat
mesh-wants' pipeline-wanted of a newer pipeline on that branch (its wants written to the end,
tier-late included) frees that record from every revision below the newer one, for each module the
newer revision's movers name. The indexer consumes pipeline-wanted (its manifest says so). A
revision a release pin holds keeps its entries, as removeUnlisted already does for any kept revision.

Tests: rows in the pins table (a passed pipeline's revision kept once no longer current; a failed
one's goes), the freeing in the pins test (a release past a failed pipeline, freed by the next
passed one's pipeline-wanted, twice changing nothing), and a pipeline-wanted row in the events
test. TestIndexerWhole now reads main.11's release beside its validate base, and waits for the held
source-moved's acknowledgement before it reads the consumer (the write comes first). No test added.

Builds: design 47 H-A-168, H-A-170 (hq design/47-H1-answers-59 87b43ec8), H-A-147, H §749
This commit is contained in:
2026-11-20 17:50:08 +01:00
parent 20f0e9baaf
commit 9083287d57
9 changed files with 190 additions and 28 deletions
@@ -303,8 +303,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
t.Cleanup(cancel) t.Cleanup(cancel)
// The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register // The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed // (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed,
// and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them. // the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted
// (H-A-170), as the grants part makes them.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -313,7 +314,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
} }
for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged, for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged,
meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll, meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll,
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} { meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
// or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked. // or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked.
for _, row := range []struct { for _, row := range []struct {
name, body string name, body string
ended bool ended, wanted bool
err error err error
acked, termd int acked, termd int
started, endedOf string started, endedOf string
}{ }{
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
{name: "not json", body: `nope`, termd: 1}, {name: "not json", body: `nope`, termd: 1},
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
} { } {
ix := &fakeRuns{err: row.err} ix := &fakeRuns{err: row.err}
m := &fakeMsg{subject: "x", data: []byte(row.body)} m := &fakeMsg{subject: "x", data: []byte(row.body)}
if row.ended { switch {
case row.wanted:
heardPipelineWanted(context.Background(), ix, m, t.Logf)
if ix.wanted != "main.12" {
t.Errorf("%s: freed the releases below %q, want main.12", row.name, ix.wanted)
}
case row.ended:
heardPipelineEnded(context.Background(), ix, m, t.Logf) heardPipelineEnded(context.Background(), ix, m, t.Logf)
} else { default:
heardRunStarted(context.Background(), ix, m, t.Logf) heardRunStarted(context.Background(), ix, m, t.Logf)
} }
if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf { if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf {
@@ -139,8 +146,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
// fakeRuns records the run events handed to it and answers err. // fakeRuns records the run events handed to it and answers err.
type fakeRuns struct { type fakeRuns struct {
err error err error
started, ended string started, ended, wanted string
} }
func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error { func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error {
@@ -157,4 +164,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
return nil return nil
} }
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64) error {
f.wanted = fmt.Sprintf("%s.%d", branch, source)
return f.err
}
func (f *fakeRuns) JudgePins(context.Context) {} func (f *fakeRuns) JudgePins(context.Context) {}
@@ -6,15 +6,17 @@ import (
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/process"
) )
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on // The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, // pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable
// and only then acknowledged, so one said while the indexer was down is applied when it comes back. // consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then
// acknowledged, so one said while the indexer was down is applied when it comes back.
// pinsStreams are the streams of the runner's events the indexer's pins follow. // pinsStreams are the streams of the events the indexer's pins follow.
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded} var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted}
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet. // pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) { func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
@@ -8,17 +8,20 @@ import (
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index" "mesh-delivery-indexer/internal/index"
) )
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline), // runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins). // and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted,
// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins).
type runTaker interface { type runTaker interface {
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
HeardPipelineWanted(ctx context.Context, branch string, source int64) error
JudgePins(ctx context.Context) JudgePins(ctx context.Context)
} }
@@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
{meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }},
} { } {
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error { _ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf) cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf)
@@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
ix.JudgePins(ctx) ix.JudgePins(ctx)
} }
// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late
// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168,
// H-A-170).
func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
var e meshwants.PipelineWanted
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 {
logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source)
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
if err == nil {
ix.JudgePins(ctx)
}
}
func settleRun(m delivered, err error, what string, logf func(string, ...any)) { func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
switch { switch {
case err == nil: case err == nil:
@@ -7,6 +7,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"os" "os"
"slices"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -345,11 +346,15 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende
} }
} }
// pinned is whether a pins value names the record. // pinned is whether a pins value names exactly the records, in any order.
func pinned(record string) func([]byte) bool { func pinned(records ...string) func([]byte) bool {
return func(v []byte) bool { return func(v []byte) bool {
var p index.PinValue var p index.PinValue
return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record if v == nil || json.Unmarshal(v, &p) != nil {
return false
}
slices.Sort(p.Records)
return slices.Equal(p.Records, slices.Sorted(slices.Values(records)))
} }
} }
@@ -484,7 +489,9 @@ func TestIndexerWhole(t *testing.T) {
return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() && return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() &&
hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13 hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13
}) })
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held. // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the
// write, so it is waited for before the consumer is read.
heard.acked(t, "main")
if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil { if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil {
t.Fatal(err) t.Fatal(err)
} else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 { } else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 {
@@ -496,8 +503,9 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
} }
untilKey(t, ctx, held, keys.EscapeName("main"), absent) untilKey(t, ctx, held, keys.EscapeName("main"), absent)
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147) // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
if n := len(heard.pending); n != 0 { if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
} }
@@ -20,8 +20,8 @@ import (
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153, // The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch // §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each // and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each
// running pipeline's (design 47 H §743, §749). // running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170).
const ( const (
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB). // PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
PinsState = "pins" PinsState = "pins"
@@ -84,6 +84,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
case started != nil && started.Revision > 0 && started.Pipeline != "": case started != nil && started.Revision > 0 && started.Pipeline != "":
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
case ended != nil && ended.Revision > 0 && ended.Pipeline != "": case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
// A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed,
// one record per mover (H-A-168, H-A-170); a run on request's pipeline delivers nothing and pins none.
if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" {
for _, m := range ended.Movers {
if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil {
return err
}
}
}
record := PipelineRecord(ended.Pipeline) record := PipelineRecord(ended.Pipeline)
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil { if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
return err return err
@@ -93,6 +102,56 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
return nil return nil
} }
// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release.<branch>.<module>`,
// each name escaped as a key's name is (design 47 H-A-168, H-A-170).
func ReleaseRecord(branch, module string) string {
return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module)
}
// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said once when a pipeline's wants are written to the end,
// tier-late included (design 47 H §751, H-A-168, H-A-170): the newer release of each module the pipeline's kept revision names
// as a mover is the want everywhere, so that module's release record is freed from every revision of the branch below it. A
// pipeline whose revision is not kept frees nothing and says so; one applied twice changes nothing.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64) error {
if ix.Pins == nil || ix.Revisions == nil {
return errors.New("the indexer keeps no state " + PinsState)
}
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
if err != nil {
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
return nil
}
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return err
}
for _, k := range pinKeys {
b, r, ok := ParsePinKey(k)
if !ok || b != branch || r >= rev.Revision {
continue
}
for _, m := range rev.Movers {
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return err
}
var pv PinValue
record := ReleaseRecord(branch, m.Module)
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
continue
}
if err := ix.unpin(ctx, k, record); err != nil {
return err
}
}
}
return nil
}
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the // unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
// same pipeline (H-A-147), which the pipeline-ended does not name. // same pipeline (H-A-147), which the pipeline-ended does not name.
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error { func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
@@ -92,6 +92,44 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
t.Errorf("the state holds %v after the pin drop, want no main.1", keys) t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
} }
// H-A-168, H-A-170: a passed pipeline's release keeps its revision past a later failed pipeline; the next passed pipeline's
// pipeline-wanted (its wants written to the end) frees the older release of the module it moves, and the revision goes.
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
take := func(source int64) {
t.Helper()
e := reader
e.Commit = fmt.Sprintf("r%d", source)
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil {
t.Fatal(err)
}
}
end := func(source int64, outcome string) {
t.Helper()
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
if err := ix.HeardPins(ctx, nil, &e); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
take(10)
end(10, runner.PipelineEndedOutcomePassed)
take(11)
end(11, runner.PipelineEndedOutcomeFailed)
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed)
want("two releases, the one between them gone", "main.10", "main.12")
for range 2 { // heard twice: changes nothing more
if err := ix.HeardPipelineWanted(ctx, "main", 12); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
want("the newer release written everywhere", "main.12")
if v, _, _, _ := ix.Pins.Get(ctx, "main.12"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.12 holds %s, want its release alone", v)
}
} }
func mustRecords(t *testing.T, v []byte) []string { func mustRecords(t *testing.T, v []byte) []string {
@@ -107,7 +145,8 @@ func mustRecords(t *testing.T, v []byte) []string {
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept // H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline // before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only // runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
// its own and names no base; the pipeline's end frees both. // its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release
// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current.
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
ctx := context.Background() ctx := context.Background()
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
@@ -117,11 +156,18 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
before []int64 // sources taken in with no mover before the take-in of source 5 before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string whileRuns []string
afterTheEnd []string afterTheEnd []string
base int64 // the base main.5 names (H-A-154) base int64 // the base main.5 names (H-A-154)
passed bool // main.5's pipeline passed: its release pins main.5
then []int64 // sources taken in with no mover after the end
released []string
}{ }{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
@@ -156,17 +202,29 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
} }
} }
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil { end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f"}}}
if tc.passed {
end.Outcome = runner.PipelineEndedOutcomePassed
}
if err := ix.HeardPins(ctx, nil, &end); err != nil {
t.Fatal(err) t.Fatal(err)
} }
ix.JudgePins(ctx) ix.JudgePins(ctx)
for _, s := range tc.then {
keep(s, nil)
}
if got := kept(); !slices.Equal(got, tc.afterTheEnd) { if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd) t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
} }
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 { if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
for _, k := range keys { for _, k := range keys {
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 { want := []string{}
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v) if slices.Contains(tc.released, k) {
want = []string{ReleaseRecord("main", "reader")}
}
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) {
t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want)
} }
} }
} }
@@ -170,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// internal/broker/state.go:62-93). // internal/broker/state.go:62-93).
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked", if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want) t.Errorf("consumes = %v, want %v", m.Consumes, want)
} }
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -89,6 +89,7 @@
"mesh-pipeline-runner.run-started", "mesh-pipeline-runner.run-started",
"mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-ended",
"mesh-wants.pipeline-wanted",
"node-engine.running", "node-engine.running",
"operator-channel.decided", "operator-channel.decided",
"npm-package-registry.published" "npm-package-registry.published"