A batched take-in names only its newest revision's movers; the other modules it moved never got a pipeline. Starter.TakenIn and the runner's first-half pickup now start one for each module of the branch's head with no pipeline at its fingerprint. Builds: design 47 G §44, §54, §88, §476
317 lines
16 KiB
Go
317 lines
16 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
)
|
|
|
|
type created map[string][]byte
|
|
|
|
func (c created) Create(key string, value []byte) error {
|
|
if _, ok := c[key]; ok {
|
|
return ErrExists
|
|
}
|
|
c[key] = value
|
|
return nil
|
|
}
|
|
|
|
// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start
|
|
// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names
|
|
// behind by kind 1, keyed by the publishing branch and revision.
|
|
func TestStarts(t *testing.T) {
|
|
store := created{}
|
|
var said []string
|
|
var superseding []string
|
|
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
|
|
ev, started := body.(seat.PipelineStarted)
|
|
if !started { // a pipeline's pin of its catalog graph revision (A-449) is no pipeline-started
|
|
return nil
|
|
}
|
|
said = append(said, subject+" "+ev.Pipeline)
|
|
if ev.Published == nil || ev.Published.Artifact != "mesh-sdk:go" || ev.Published.Version != "0.1.16" {
|
|
t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published)
|
|
}
|
|
return nil
|
|
}, Supersede: func(newer seat.Pipeline) { superseding = append(superseding, newer.Pipeline) }}
|
|
view := func(branch string) (Half, bool) {
|
|
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
|
|
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
|
|
}
|
|
var asked catalog.OutdatedInput
|
|
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
|
|
asked = in
|
|
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{
|
|
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
|
|
}
|
|
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
|
|
if err != nil || n != 2 {
|
|
t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err)
|
|
}
|
|
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
|
|
t.Errorf("outdated asked %+v, want the package on its branch", asked)
|
|
}
|
|
var p seat.Pipeline
|
|
if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fingerprint != "src2:aa" || p.Revision != 9 ||
|
|
p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 || p.Published == nil || p.Published.Version != "0.1.16" {
|
|
t.Errorf("the pipeline main.41.app = %+v (%v)", p, err)
|
|
}
|
|
// Each pipeline written supersedes the older ones it replaces (§31).
|
|
if len(superseding) != 2 {
|
|
t.Errorf("the publish's pipelines superseding: %v, want both", superseding)
|
|
}
|
|
if _, ok := store["main.41.ordered"]; ok {
|
|
t.Error("an ordering edge starts no pipeline")
|
|
}
|
|
// The take-in of the same source revision and module is the same pipeline: the second start writes nothing.
|
|
before := string(store["main.41.app"])
|
|
ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"})
|
|
if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 {
|
|
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
|
|
}
|
|
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") {
|
|
t.Errorf("pipeline-started said on %q", said[0])
|
|
}
|
|
s.Say = func(string, any) error { return nil }
|
|
// A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused.
|
|
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" {
|
|
t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app"))
|
|
}
|
|
if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil {
|
|
t.Error("a start with no source revision is refused")
|
|
}
|
|
s.Pipelines = failing{}
|
|
if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil {
|
|
t.Error("a write the bus refuses is an error")
|
|
}
|
|
}
|
|
|
|
// memCreated is memStore with the Create a start writes through.
|
|
type memCreated struct{ memStore }
|
|
|
|
func (m memCreated) Create(key string, value []byte) error {
|
|
if _, ok := m.memStore[key]; ok {
|
|
return ErrExists
|
|
}
|
|
var p seat.Pipeline
|
|
if err := json.Unmarshal(value, &p); err != nil {
|
|
return err
|
|
}
|
|
m.memStore[key] = struct {
|
|
p seat.Pipeline
|
|
rev uint64
|
|
}{p, 1}
|
|
return nil
|
|
}
|
|
|
|
type failing struct{}
|
|
|
|
func (failing) Create(string, []byte) error { return errors.New("bus away") }
|
|
|
|
// A take-in.taken starts one pipeline per entry of its source revision (design 47 G §44, §54, §476); a hand drop starts
|
|
// none, and a publish that started an entry first leaves the take-in's start of it writing nothing.
|
|
func TestTakenInStarts(t *testing.T) {
|
|
store := created{}
|
|
var said []string
|
|
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
|
|
if _, pins := body.(delivery.Pins); !pins {
|
|
said = append(said, subject)
|
|
}
|
|
return nil
|
|
}}
|
|
view := func(string) (Half, bool) { return Half{Revision: 12}, true }
|
|
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{
|
|
{Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}}
|
|
if _, err := s.Start(Begin{Branch: "main", Source: 7, Module: "b", Commit: "c2"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := s.TakenIn(ev, view)
|
|
if err != nil || n != 1 || len(said) != 2 {
|
|
t.Fatalf("a take-in started %d (%v), said %v: want a only, b being started already", n, err, said)
|
|
}
|
|
var p seat.Pipeline
|
|
if err := json.Unmarshal(store["main.7.a"], &p); err != nil || p.Commit != "c1" || p.Group != "g1" || p.Revision != 12 || p.Fingerprint != "src2:aa" {
|
|
t.Errorf("the pipeline main.7.a = %+v (%v)", p, err)
|
|
}
|
|
if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Dropped: true, By: "op", Branch: "main", Source: 8, Entries: ev.Entries}, view); n != 0 || err != nil {
|
|
t.Errorf("a hand drop started %d (%v)", n, err)
|
|
}
|
|
if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main", Entries: ev.Entries}, view); err == nil {
|
|
t.Error("a take-in naming no source revision is refused")
|
|
}
|
|
// §493: a taken event with no entries (a group's end, one said before G) starts nothing and is not heard again.
|
|
if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Key: "m.9"}, view); n != 0 || err != nil {
|
|
t.Errorf("a take-in with no entries started %d (%v)", n, err)
|
|
}
|
|
}
|
|
|
|
// A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the
|
|
// registration-changed that brings it places that pipeline's steps from it and asks them, once: heard again it places nothing.
|
|
func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T) {
|
|
reg, entries := placeFixture()
|
|
store := memCreated{memStore{}}
|
|
asks := &stepAsks{}
|
|
registered := false
|
|
var said []string
|
|
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(subject string, _ any) error { said = append(said, subject); return nil },
|
|
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
|
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
|
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
|
|
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
|
t.Fatalf("the take-in started %d (%v)", n, err)
|
|
}
|
|
if p, _, _ := store.Get("main.4.app"); len(p.Steps) != 0 || len(asks.asks) != 0 {
|
|
t.Fatalf("before its registration the pipeline has %d steps and %d asks, want none", len(p.Steps), len(asks.asks))
|
|
}
|
|
registered = true
|
|
n, err := s.Registered(store.memStore, reg, view)
|
|
p, _, _ := store.Get("main.4.app")
|
|
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Outcome != seat.PipelineOutcomeRunning {
|
|
t.Fatalf("the registration placed %d (%v): %d steps, %d asks, want the pipeline's steps placed and asked", n, err, len(p.Steps), len(asks.asks))
|
|
}
|
|
before := len(asks.asks)
|
|
if n, err := s.Registered(store.memStore, reg, view); n != 0 || err != nil || len(asks.asks) != before {
|
|
t.Errorf("the registration heard again placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before)
|
|
}
|
|
|
|
// The registration heard before the branch's catalog half is held (or before the runner could place from it) is not
|
|
// lost: the half becoming held, or the registrations' first view, places the waiting pipeline once (Waiting).
|
|
held := false
|
|
late := func(string) (Half, bool) { return Half{Entries: entries}, held }
|
|
taken = catalog.TakeInTaken{Key: "m.2", Branch: "next", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}}
|
|
registered = false
|
|
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
|
t.Fatalf("the take-in on next started %d (%v)", n, err)
|
|
}
|
|
registered = true
|
|
if n, err := s.Registered(store.memStore, reg, late); n != 0 || err != nil {
|
|
t.Fatalf("the registration before the half placed %d (%v), want none and no error: it waits on the half", n, err)
|
|
}
|
|
before = len(asks.asks)
|
|
held = true
|
|
if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil {
|
|
t.Errorf("the half of main placed %d (%v) of next's pipelines", n, err)
|
|
}
|
|
n, err = s.Waiting(store.memStore, "next", late)
|
|
p, _, _ = store.Get("next.5.app")
|
|
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
|
|
t.Fatalf("the half of next becoming held placed %d (%v): %d steps, %d asks more, want its steps placed and asked", n, err, len(p.Steps), len(asks.asks)-before)
|
|
}
|
|
before = len(asks.asks)
|
|
if n, err := s.Waiting(store.memStore, "", late); n != 0 || err != nil || len(asks.asks) != before {
|
|
t.Errorf("the registrations' first view after it placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before)
|
|
}
|
|
|
|
// A-336 (§200): a take-in heard before its branch's half is held starts its pipelines with no steps, never placed from
|
|
// empty entries, the registration held or not; the half becoming held places them (Waiting).
|
|
held = false
|
|
taken = catalog.TakeInTaken{Key: "m.3", Branch: "next", Source: 6, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:cc"}}}
|
|
if n, err := s.TakenIn(taken, late); n != 1 || err != nil {
|
|
t.Fatalf("the take-in before the half started %d (%v)", n, err)
|
|
}
|
|
if p, _, _ := store.Get("next.6.app"); len(p.Steps) != 0 {
|
|
t.Fatalf("a take-in before its branch's half placed %d steps from empty entries, want none", len(p.Steps))
|
|
}
|
|
held = true
|
|
n, err = s.Waiting(store.memStore, "next", late)
|
|
p, _, _ = store.Get("next.6.app")
|
|
if n != 1 || err != nil || len(p.Steps) == 0 {
|
|
t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps))
|
|
}
|
|
// A-336: so does a package published heard before its branch's half is held.
|
|
held = false
|
|
outdated := func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
|
|
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{{Module: "app", Kind: outdatedKind}}}, nil
|
|
}
|
|
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}, outdated, late); n != 1 || err != nil {
|
|
t.Fatalf("the publish before the half started %d (%v)", n, err)
|
|
}
|
|
if p, _, _ := store.Get("next.7.app"); len(p.Steps) != 0 {
|
|
t.Fatalf("a publish before its branch's half placed %d steps from empty entries, want none", len(p.Steps))
|
|
}
|
|
held = true
|
|
n, err = s.Waiting(store.memStore, "next", late)
|
|
p, _, _ = store.Get("next.7.app")
|
|
if n != 1 || err != nil || len(p.Steps) == 0 {
|
|
t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps))
|
|
}
|
|
|
|
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
|
|
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
|
|
var old seat.Pipeline
|
|
if err := json.Unmarshal([]byte(`{"pipeline":"main.940.app","branch":"main","module":"app","commit":"c9","source":940,"outcome":"running",`+
|
|
`"held":{"since":"2026-11-15T18:05:00Z","fingerprint":"src2:new"}}`), &old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store.memStore["main.940.app"] = struct {
|
|
p seat.Pipeline
|
|
rev uint64
|
|
}{old, 1}
|
|
before, saidBefore := len(asks.asks), len(said)
|
|
n, err = s.Waiting(store.memStore, "main", late)
|
|
p, _, _ = store.Get("main.940.app")
|
|
if len(said) != saidBefore+1 || !strings.HasPrefix(said[saidBefore], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c9") {
|
|
t.Errorf("placing the set-aside pipeline said %v, want its pipeline-started (A-446a)", said[saidBefore:])
|
|
}
|
|
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
|
|
t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before)
|
|
}
|
|
}
|
|
|
|
// A batch of merges taken in together is one take-in.taken naming only the newest revision's movers: every other module of the
|
|
// branch's head with no pipeline at its fingerprint gets one too, at the take-in's source revision; a module whose fingerprint
|
|
// has a pipeline already (of any source revision) and a run on request's pipeline start nothing.
|
|
func TestTakenInStartsEveryModuleWithNoPipelineAtItsFingerprint(t *testing.T) {
|
|
mem := memStore{}
|
|
store := memCreated{mem}
|
|
s := &Starter{Pipelines: store, Store: mem, Say: func(string, any) error { return nil }}
|
|
half := Half{Revision: 12, Source: 9, Entries: map[string]catalog.CatalogEntry{
|
|
"moved": {Commit: "c1", Fp: "src2:aa", Repository: "o/moved"},
|
|
"batched": {Commit: "c2", Fp: "src2:bb", Repository: "o/batched"},
|
|
"done": {Commit: "c3", Fp: "src2:cc"},
|
|
"run": {Commit: "c4", Fp: "src2:dd"},
|
|
"nofp": {Commit: "c5"},
|
|
}}
|
|
view := func(string) (Half, bool) { return half, true }
|
|
mem["main.3.done"] = struct {
|
|
p seat.Pipeline
|
|
rev uint64
|
|
}{seat.Pipeline{Module: "done", Fingerprint: "src2:cc", Outcome: seat.PipelineOutcomePassed}, 1}
|
|
mem["main.4.run"] = struct {
|
|
p seat.Pipeline
|
|
rev uint64
|
|
}{seat.Pipeline{Module: "run", Fingerprint: "src2:dd", Run: "r1"}, 1}
|
|
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12,
|
|
Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}}
|
|
n, err := s.TakenIn(ev, view)
|
|
if err != nil || n != 3 {
|
|
t.Fatalf("the take-in started %d (%v), want moved, batched and run (a run on request is no pipeline of the head)", n, err)
|
|
}
|
|
for _, k := range []string{"main.9.moved", "main.9.batched", "main.9.run"} {
|
|
if _, _, err := mem.Get(k); err != nil {
|
|
t.Errorf("%s is not started: %v", k, err)
|
|
}
|
|
}
|
|
for _, k := range []string{"main.9.done", "main.9.nofp"} {
|
|
if _, _, err := mem.Get(k); err == nil {
|
|
t.Errorf("%s is started, but its module has a pipeline at its fingerprint or none", k)
|
|
}
|
|
}
|
|
if n, err := s.TakenIn(ev, view); err != nil || n != 0 {
|
|
t.Errorf("the same take-in heard again started %d (%v), want none", n, err)
|
|
}
|
|
// A half naming no source revision starts none (the start-up sweep reads the half's own).
|
|
if n, err := s.StartUnstarted("main", 0, Half{Entries: map[string]catalog.CatalogEntry{"late": {Fp: "src2:ee"}}}); n != 0 || err != nil {
|
|
t.Errorf("a half with no source revision started %d (%v)", n, err)
|
|
}
|
|
}
|