Join: merge feat/47-H2-f3 into feat/47-H2
This commit is contained in:
@@ -22,8 +22,10 @@ import (
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
@@ -53,13 +55,25 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}}
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "required", Approval: "bypass"}}}
|
||||
// design 47 H2 §790: the stalled bound is the setting stalled-bound, here one minute, read as the node-engine composes it.
|
||||
t.Setenv(stdio.SettingWord(pipeline.StalledBoundSetting), "1m")
|
||||
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336).
|
||||
var revised atomic.Bool
|
||||
w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer {
|
||||
if revised.Load() {
|
||||
// design 47 H2 §775, H-A-453 (3): the kept-call pace a hundredth of its default (10 ms doubling to 300 ms), so the
|
||||
// holder away for 40 s at the default pace is away for 400 ms here.
|
||||
const away = 400 * time.Millisecond
|
||||
t.Cleanup(busretry.Waits(busretry.Bound, 10*time.Millisecond, 300*time.Millisecond))
|
||||
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336), then the
|
||||
// re-pin revision 5 of the same source moving app to src2:ab (H-A-283), its newest once said.
|
||||
var revised, repinned atomic.Bool
|
||||
w := startWhole(t, ctx, []catalog.Registration{reg}, func(raw []byte) catalog.RevisionAnswer {
|
||||
var in catalog.RevisionInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
switch {
|
||||
case repinned.Load() && (in.Revision == 5 || in.Revision == 0):
|
||||
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 5, Kind: "trunk", Branch: "main", Source: 3, CallKind: catalog.CatalogRevisionCallKindRelease,
|
||||
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}},
|
||||
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:ab"}}}
|
||||
case revised.Load():
|
||||
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3},
|
||||
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"}}}
|
||||
}
|
||||
@@ -67,6 +81,17 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
})
|
||||
const module = "mesh-delivery-pipeline"
|
||||
nc, js, asks, pipelines, logs := w.nc, w.js, w.asks, w.pipelines, w.logs
|
||||
// design 47 H2 §821: every event the runner says is waited on by its seat and name, as mesh_wait waits on
|
||||
// mesh-pipeline-runner.<event>; each is heard by the test's end.
|
||||
heard := map[string][][]byte{}
|
||||
var heardMu sync.Mutex
|
||||
for name, subject := range map[string]string{seat.EventPipelineStarted: seat.SubjectPipelineStartedAll, seat.EventStepEnded: seat.SubjectStepEndedAll,
|
||||
seat.EventPipelineEnded: seat.SubjectPipelineEndedAll, seat.EventRePinned: seat.SubjectRePinned} {
|
||||
if _, err := nc.Subscribe(subject, func(m *nats.Msg) { heardMu.Lock(); heard[name] = append(heard[name], m.Data); heardMu.Unlock() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
heardOf := func(name string) [][]byte { heardMu.Lock(); defer heardMu.Unlock(); return heard[name] }
|
||||
|
||||
// A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module.
|
||||
taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: 3, Call: "0123456789abcdef",
|
||||
@@ -103,6 +128,47 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
|
||||
t.Fatalf("the build step = %+v", build)
|
||||
}
|
||||
// A re-pin revision of the pipeline's own source keeps app in it at src2:ab, its steps placed again under new asks, and
|
||||
// says re-pinned naming the pipeline, the mover and the revision (design 47 H2 §821, H-A-283).
|
||||
// The holder of mesh-wants is away (§775): no work queue takes its asks, so the composition asked again meets no
|
||||
// responders and is put again at the kept-call pace.
|
||||
if _, err := js.UpdateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{builds.AcceptSubjectStep}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repinned.Store(true)
|
||||
repin, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 5, Kind: "trunk", Branch: "main", Source: 3, CallKind: catalog.CatalogRevisionEventCallKindRelease})
|
||||
if _, err := js.Publish(ctx, catalog.SubjectCatalogRevision("main"), repin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return len(heardOf(seat.EventRePinned)) > 0 })
|
||||
var said seat.RePinnedEvent
|
||||
if err := json.Unmarshal(heardOf(seat.EventRePinned)[0], &said); err != nil || said.Pipeline != key || said.Mover != "app" || said.Revision != 5 {
|
||||
t.Fatalf("re-pinned said %+v (%v), want %s app 5", said, err, key)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Fingerprint == "src2:ab"
|
||||
})
|
||||
if build = stepOf(p, seat.StepResultStepBuild); build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
|
||||
t.Fatalf("the build placed again = %+v", build)
|
||||
}
|
||||
// The composition ask meets the holder away, said in the journal once; the holder is back after the away span and the
|
||||
// ask lands on its work queue, no hand act between (§775).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return strings.Contains(logs.String(), "kept call: the seat "+wants.Seat+" does not answer")
|
||||
})
|
||||
gone := time.Now()
|
||||
waitFor(t, ctx, logs, func() bool { return time.Since(gone) >= away })
|
||||
if _, err := js.UpdateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := stepOf(p, seat.StepResultStepComposition)
|
||||
if composition.Outcome != seat.StepResultOutcomeRunning || composition.Ask == "" || composition.Fingerprint != "src2:ab" {
|
||||
t.Fatalf("the composition asked again = %+v", composition)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
m, err := asks.GetLastMsgForSubject(ctx, wants.AcceptSubjectStep)
|
||||
return err == nil && m.Header.Get(nats.MsgIdHdr) == composition.Ask
|
||||
})
|
||||
// Once a step is placed in it, the health no longer names it (§790).
|
||||
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); strings.Contains(a.Why, key+" (nothing open") {
|
||||
t.Fatalf("with its build placed, the health still names %s: %+v", key, a)
|
||||
@@ -111,7 +177,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 })
|
||||
|
||||
// A session's review, while the build still runs, is answered by the runner's process, the caller the one the subject names.
|
||||
in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"})
|
||||
in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Outcome: "passed", Said: "reads well"})
|
||||
var reply struct {
|
||||
Result seat.ReviewAnswer `json:"result"`
|
||||
Error string `json:"error"`
|
||||
@@ -131,7 +197,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
|
||||
// The seat's holder says the build ended; the runner takes it from the node's subject.
|
||||
ended, _ := json.Marshal(builds.StepEnded{ID: build.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
|
||||
Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
|
||||
Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab"}}})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -139,7 +205,20 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Outcome == seat.StepResultOutcomePassed
|
||||
})
|
||||
|
||||
// The holder back says the composition ended passed: the step passes, the pipeline with it.
|
||||
composed, _ := json.Marshal(wants.StepEnded{ID: composition.Ask, Pipeline: key, Step: wants.StepRunStepComposition, Node: "n1", Outcome: "passed",
|
||||
Module: "app", Commit: "c1", Fingerprint: "src2:ab"})
|
||||
if _, err := js.Publish(ctx, wants.SubjectStepEnded("n1"), composed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
|
||||
if c := stepOf(p, seat.StepResultStepComposition); c.Outcome != seat.StepResultOutcomePassed || c.Ask != composition.Ask {
|
||||
t.Fatalf("the composition whose holder was away ended %+v, want passed under its ask %s", c, composition.Ask)
|
||||
}
|
||||
// Each of the runner's events was heard by its name (§821).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return len(heardOf(seat.EventPipelineStarted)) > 0 && len(heardOf(seat.EventStepEnded)) > 0 && len(heardOf(seat.EventPipelineEnded)) > 0
|
||||
})
|
||||
|
||||
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
|
||||
// record written before it, its steps inline, is read whole and split at its next write.
|
||||
|
||||
@@ -131,6 +131,7 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
}
|
||||
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
|
||||
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
|
||||
s.sayRePinned(p, kept, revisions, half.Revision)
|
||||
for _, st := range ended {
|
||||
if s.Say == nil {
|
||||
break
|
||||
@@ -164,6 +165,23 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
}
|
||||
}
|
||||
|
||||
// sayRePinned says re-pinned for each of kept whose fingerprint the re-pin revision moved (it is in revisions), once its
|
||||
// pipeline p is written keeping it (design 47 H2 §821, H-A-349 DQ-13): the pipeline, the mover and the revision it is
|
||||
// re-pinned at, on the seat's subject for the event, so a wait by the seat and the event's name is answered.
|
||||
func (s *Starter) sayRePinned(p seat.Pipeline, kept []seat.Mover, revisions map[string]int64, revision int64) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
for _, m := range kept {
|
||||
if _, ok := revisions[m.Module]; !ok {
|
||||
continue
|
||||
}
|
||||
if err := s.Say(seat.SubjectRePinned, seat.RePinnedEvent{Pipeline: p.Pipeline, Mover: m.Module, Revision: revision}); err != nil {
|
||||
s.logf("%s of %s's %s at revision %d is not said: %v", seat.EventRePinned, p.Pipeline, m.Module, revision, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// placeAgain places the steps of kept mover m of p from the re-pin revision half's entry (H-A-284 (1)), as a mover is placed
|
||||
// (placeMovers), under ask ids its kept revision is mixed into (H-A-286): it answers the asks to put, the steps placed not asked because a
|
||||
// dependency did not build, and whether m has no registration. A mover not due yet (it waits on a publish) is left
|
||||
|
||||
@@ -66,13 +66,17 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var repinned []string
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
Say: func(_ string, body any) error {
|
||||
Say: func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.StepEvent); ok && ev.Result.Outcome == seat.StepResultOutcomeSuperseded {
|
||||
said = append(said, ev)
|
||||
}
|
||||
if ev, ok := body.(seat.RePinnedEvent); ok {
|
||||
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
|
||||
@@ -122,6 +126,10 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if !ok || asks.ids[0] != p.Steps[0].Ask || b.Revision != 10 || b.Source != 41 || len(b.Entries) != 1 || b.Entries[0].Fingerprint != "src2:ab" {
|
||||
t.Fatalf("the build asked again = %+v (%s), want the new fingerprint at the pipeline's revision", asks.asks[0], asks.ids[0])
|
||||
}
|
||||
// design 47 H2 §821: the kept mover's re-pin is said once, on the seat's re-pinned, naming the pipeline, mover and revision.
|
||||
if want := seat.SubjectRePinned + " " + key + " app 11"; len(repinned) != 1 || repinned[0] != want {
|
||||
t.Errorf("re-pinned said %q, want once %q", repinned, want)
|
||||
}
|
||||
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
|
||||
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
|
||||
// Design 47 H2 §799: a fetch met by a forge's 401 or a connection fault, and a publish met by no responders or a bundle
|
||||
// that exited, end their run `again` after again-delays' delay, with no seat call; a third run's fault ends the step failed
|
||||
// with the why `fetch: <words>` or `publish: <words>`.
|
||||
// with the why `fetch: <words>` or `publish: <words>`. ADR 0664 §1 with the bus's count above the runs that started: an ask
|
||||
// given back twice before its first run (the bus says delivered 3, the step key 1 run began) meets its fault in run 2,
|
||||
// given again after again-delays' second, 60 s, not 3.
|
||||
func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
|
||||
delay, err := AgainDelays("")
|
||||
if err != nil || delay(1) != 15*time.Second || delay(2) != time.Minute || delay(3) != time.Minute {
|
||||
@@ -46,8 +48,14 @@ func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
|
||||
if !sdkstep.IsPassing(tc.err) || !strings.HasPrefix(tc.err.Error(), tc.why) {
|
||||
t.Fatalf("%v: want a transport fault opening with %q", tc.err, tc.why)
|
||||
}
|
||||
for _, prior := range []int{0, 2} { // its first run ends again after 15 s; its third ends the step failed
|
||||
// Its first run ends again after 15 s; its second, delivered three times, again after 60 s; its third ends the step failed.
|
||||
for _, run := range []struct {
|
||||
prior int
|
||||
delivered uint64
|
||||
}{{0, 0}, {1, 3}, {2, 0}} {
|
||||
prior := run.prior
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = run.delivered
|
||||
a := testAsk(sdkstep.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
@@ -65,6 +73,9 @@ func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
|
||||
if prior == 0 && (s.State != sdkstep.StepAgain || len(delays) != 1 || delays[0] != 15*time.Second) {
|
||||
t.Errorf("the first run: step %+v, delays %v, want again after 15s", s, delays)
|
||||
}
|
||||
if prior == 1 && (s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != time.Minute) {
|
||||
t.Errorf("the second run, delivered %d times: step %+v, delays %v, want again in run 2 after 60s", run.delivered, s, delays)
|
||||
}
|
||||
if prior == 2 && (s.State != sdkstep.StepFailed || !strings.HasPrefix(s.Reason, tc.why)) {
|
||||
t.Errorf("the third run: step %+v, want failed with the why %q", s, tc.why)
|
||||
}
|
||||
|
||||
@@ -246,33 +246,6 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the
|
||||
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after again-delays'
|
||||
// second, 60 s, design 47 H2 §799), not 3.
|
||||
func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = 3
|
||||
a := testAsk(sdkstep.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
|
||||
Prior: func(string) int { return 1 },
|
||||
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
|
||||
return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 })
|
||||
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
|
||||
s, _ := st.get(key)
|
||||
src.mu.Lock()
|
||||
delays := append([]time.Duration(nil), src.Delays...)
|
||||
src.mu.Unlock()
|
||||
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != time.Minute {
|
||||
t.Errorf("step %+v, delays %v, want again in run 2 after 60s", s, delays)
|
||||
}
|
||||
}
|
||||
|
||||
// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no
|
||||
// step, and nil for no read.
|
||||
func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user