Join: merge feat/47-H2-f3 into feat/47-H2

This commit is contained in:
2026-11-22 03:39:45 +01:00
5 changed files with 126 additions and 37 deletions
@@ -22,8 +22,10 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -53,13 +55,25 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}}
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "required", Approval: "bypass"}}}
// design 47 H2 §790: the stalled bound is the setting stalled-bound, here one minute, read as the node-engine composes it.
t.Setenv(stdio.SettingWord(pipeline.StalledBoundSetting), "1m")
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336).
var revised atomic.Bool
w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer {
if revised.Load() {
// design 47 H2 §775, H-A-453 (3): the kept-call pace a hundredth of its default (10 ms doubling to 300 ms), so the
// holder away for 40 s at the default pace is away for 400 ms here.
const away = 400 * time.Millisecond
t.Cleanup(busretry.Waits(busretry.Bound, 10*time.Millisecond, 300*time.Millisecond))
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336), then the
// re-pin revision 5 of the same source moving app to src2:ab (H-A-283), its newest once said.
var revised, repinned atomic.Bool
w := startWhole(t, ctx, []catalog.Registration{reg}, func(raw []byte) catalog.RevisionAnswer {
var in catalog.RevisionInput
_ = json.Unmarshal(raw, &in)
switch {
case repinned.Load() && (in.Revision == 5 || in.Revision == 0):
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 5, Kind: "trunk", Branch: "main", Source: 3, CallKind: catalog.CatalogRevisionCallKindRelease,
Movers: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}},
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:ab"}}}
case revised.Load():
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3},
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"}}}
}
@@ -67,6 +81,17 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
})
const module = "mesh-delivery-pipeline"
nc, js, asks, pipelines, logs := w.nc, w.js, w.asks, w.pipelines, w.logs
// design 47 H2 §821: every event the runner says is waited on by its seat and name, as mesh_wait waits on
// mesh-pipeline-runner.<event>; each is heard by the test's end.
heard := map[string][][]byte{}
var heardMu sync.Mutex
for name, subject := range map[string]string{seat.EventPipelineStarted: seat.SubjectPipelineStartedAll, seat.EventStepEnded: seat.SubjectStepEndedAll,
seat.EventPipelineEnded: seat.SubjectPipelineEndedAll, seat.EventRePinned: seat.SubjectRePinned} {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) { heardMu.Lock(); heard[name] = append(heard[name], m.Data); heardMu.Unlock() }); err != nil {
t.Fatal(err)
}
}
heardOf := func(name string) [][]byte { heardMu.Lock(); defer heardMu.Unlock(); return heard[name] }
// A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module.
taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: 3, Call: "0123456789abcdef",
@@ -103,6 +128,47 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
t.Fatalf("the build step = %+v", build)
}
// A re-pin revision of the pipeline's own source keeps app in it at src2:ab, its steps placed again under new asks, and
// says re-pinned naming the pipeline, the mover and the revision (design 47 H2 §821, H-A-283).
// The holder of mesh-wants is away (§775): no work queue takes its asks, so the composition asked again meets no
// responders and is put again at the kept-call pace.
if _, err := js.UpdateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{builds.AcceptSubjectStep}, Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
repinned.Store(true)
repin, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 5, Kind: "trunk", Branch: "main", Source: 3, CallKind: catalog.CatalogRevisionEventCallKindRelease})
if _, err := js.Publish(ctx, catalog.SubjectCatalogRevision("main"), repin); err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool { return len(heardOf(seat.EventRePinned)) > 0 })
var said seat.RePinnedEvent
if err := json.Unmarshal(heardOf(seat.EventRePinned)[0], &said); err != nil || said.Pipeline != key || said.Mover != "app" || said.Revision != 5 {
t.Fatalf("re-pinned said %+v (%v), want %s app 5", said, err, key)
}
waitFor(t, ctx, logs, func() bool {
return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Fingerprint == "src2:ab"
})
if build = stepOf(p, seat.StepResultStepBuild); build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
t.Fatalf("the build placed again = %+v", build)
}
// The composition ask meets the holder away, said in the journal once; the holder is back after the away span and the
// ask lands on its work queue, no hand act between (§775).
waitFor(t, ctx, logs, func() bool {
return strings.Contains(logs.String(), "kept call: the seat "+wants.Seat+" does not answer")
})
gone := time.Now()
waitFor(t, ctx, logs, func() bool { return time.Since(gone) >= away })
if _, err := js.UpdateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
composition := stepOf(p, seat.StepResultStepComposition)
if composition.Outcome != seat.StepResultOutcomeRunning || composition.Ask == "" || composition.Fingerprint != "src2:ab" {
t.Fatalf("the composition asked again = %+v", composition)
}
waitFor(t, ctx, logs, func() bool {
m, err := asks.GetLastMsgForSubject(ctx, wants.AcceptSubjectStep)
return err == nil && m.Header.Get(nats.MsgIdHdr) == composition.Ask
})
// Once a step is placed in it, the health no longer names it (§790).
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); strings.Contains(a.Why, key+" (nothing open") {
t.Fatalf("with its build placed, the health still names %s: %+v", key, a)
@@ -111,7 +177,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 })
// A session's review, while the build still runs, is answered by the runner's process, the caller the one the subject names.
in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"})
in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Outcome: "passed", Said: "reads well"})
var reply struct {
Result seat.ReviewAnswer `json:"result"`
Error string `json:"error"`
@@ -131,7 +197,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
// The seat's holder says the build ended; the runner takes it from the node's subject.
ended, _ := json.Marshal(builds.StepEnded{ID: build.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab"}}})
if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil {
t.Fatal(err)
}
@@ -139,7 +205,20 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Outcome == seat.StepResultOutcomePassed
})
// The holder back says the composition ended passed: the step passes, the pipeline with it.
composed, _ := json.Marshal(wants.StepEnded{ID: composition.Ask, Pipeline: key, Step: wants.StepRunStepComposition, Node: "n1", Outcome: "passed",
Module: "app", Commit: "c1", Fingerprint: "src2:ab"})
if _, err := js.Publish(ctx, wants.SubjectStepEnded("n1"), composed); err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
if c := stepOf(p, seat.StepResultStepComposition); c.Outcome != seat.StepResultOutcomePassed || c.Ask != composition.Ask {
t.Fatalf("the composition whose holder was away ended %+v, want passed under its ask %s", c, composition.Ask)
}
// Each of the runner's events was heard by its name (§821).
waitFor(t, ctx, logs, func() bool {
return len(heardOf(seat.EventPipelineStarted)) > 0 && len(heardOf(seat.EventStepEnded)) > 0 && len(heardOf(seat.EventPipelineEnded)) > 0
})
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
// record written before it, its steps inline, is read whole and split at its next write.
@@ -131,6 +131,7 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
}
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
s.sayRePinned(p, kept, revisions, half.Revision)
for _, st := range ended {
if s.Say == nil {
break
@@ -164,6 +165,23 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
}
}
// sayRePinned says re-pinned for each of kept whose fingerprint the re-pin revision moved (it is in revisions), once its
// pipeline p is written keeping it (design 47 H2 §821, H-A-349 DQ-13): the pipeline, the mover and the revision it is
// re-pinned at, on the seat's subject for the event, so a wait by the seat and the event's name is answered.
func (s *Starter) sayRePinned(p seat.Pipeline, kept []seat.Mover, revisions map[string]int64, revision int64) {
if s.Say == nil {
return
}
for _, m := range kept {
if _, ok := revisions[m.Module]; !ok {
continue
}
if err := s.Say(seat.SubjectRePinned, seat.RePinnedEvent{Pipeline: p.Pipeline, Mover: m.Module, Revision: revision}); err != nil {
s.logf("%s of %s's %s at revision %d is not said: %v", seat.EventRePinned, p.Pipeline, m.Module, revision, err)
}
}
}
// placeAgain places the steps of kept mover m of p from the re-pin revision half's entry (H-A-284 (1)), as a mover is placed
// (placeMovers), under ask ids its kept revision is mixed into (H-A-286): it answers the asks to put, the steps placed not asked because a
// dependency did not build, and whether m has no registration. A mover not due yet (it waits on a publish) is left
@@ -66,13 +66,17 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
store := &MoverStore{Raw: newMemRaw()}
asks := &stepAsks{}
var said []seat.StepEvent
var repinned []string
var left [][]seat.StepResult
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
Say: func(_ string, body any) error {
Say: func(subject string, body any) error {
if ev, ok := body.(seat.StepEvent); ok && ev.Result.Outcome == seat.StepResultOutcomeSuperseded {
said = append(said, ev)
}
if ev, ok := body.(seat.RePinnedEvent); ok {
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
}
return nil
},
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
@@ -122,6 +126,10 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
if !ok || asks.ids[0] != p.Steps[0].Ask || b.Revision != 10 || b.Source != 41 || len(b.Entries) != 1 || b.Entries[0].Fingerprint != "src2:ab" {
t.Fatalf("the build asked again = %+v (%s), want the new fingerprint at the pipeline's revision", asks.asks[0], asks.ids[0])
}
// design 47 H2 §821: the kept mover's re-pin is said once, on the seat's re-pinned, naming the pipeline, mover and revision.
if want := seat.SubjectRePinned + " " + key + " app 11"; len(repinned) != 1 || repinned[0] != want {
t.Errorf("re-pinned said %q, want once %q", repinned, want)
}
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
}
@@ -12,7 +12,9 @@ import (
// Design 47 H2 §799: a fetch met by a forge's 401 or a connection fault, and a publish met by no responders or a bundle
// that exited, end their run `again` after again-delays' delay, with no seat call; a third run's fault ends the step failed
// with the why `fetch: <words>` or `publish: <words>`.
// with the why `fetch: <words>` or `publish: <words>`. ADR 0664 §1 with the bus's count above the runs that started: an ask
// given back twice before its first run (the bus says delivered 3, the step key 1 run began) meets its fault in run 2,
// given again after again-delays' second, 60 s, not 3.
func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
delay, err := AgainDelays("")
if err != nil || delay(1) != 15*time.Second || delay(2) != time.Minute || delay(3) != time.Minute {
@@ -46,8 +48,14 @@ func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
if !sdkstep.IsPassing(tc.err) || !strings.HasPrefix(tc.err.Error(), tc.why) {
t.Fatalf("%v: want a transport fault opening with %q", tc.err, tc.why)
}
for _, prior := range []int{0, 2} { // its first run ends again after 15 s; its third ends the step failed
// Its first run ends again after 15 s; its second, delivered three times, again after 60 s; its third ends the step failed.
for _, run := range []struct {
prior int
delivered uint64
}{{0, 0}, {1, 3}, {2, 0}} {
prior := run.prior
src, st := NewFakeAsks(), newSteps()
src.Delivered = run.delivered
a := testAsk(sdkstep.AskDelivery)
_ = src.PublishAsk(a)
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
@@ -65,6 +73,9 @@ func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
if prior == 0 && (s.State != sdkstep.StepAgain || len(delays) != 1 || delays[0] != 15*time.Second) {
t.Errorf("the first run: step %+v, delays %v, want again after 15s", s, delays)
}
if prior == 1 && (s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != time.Minute) {
t.Errorf("the second run, delivered %d times: step %+v, delays %v, want again in run 2 after 60s", run.delivered, s, delays)
}
if prior == 2 && (s.State != sdkstep.StepFailed || !strings.HasPrefix(s.Reason, tc.why)) {
t.Errorf("the third run: step %+v, want failed with the why %q", s, tc.why)
}
@@ -246,33 +246,6 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) {
}
}
// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after again-delays'
// second, 60 s, design 47 H2 §799), not 3.
func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
src, st := NewFakeAsks(), newSteps()
src.Delivered = 3
a := testAsk(sdkstep.AskDelivery)
_ = src.PublishAsk(a)
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond,
Prior: func(string) int { return 1 },
Run: func(context.Context, sdkstep.StepAsk) (any, error) {
return nil, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}
}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Pull(ctx)
waitFor(t, "the ask given again", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) == 1 })
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
s, _ := st.get(key)
src.mu.Lock()
delays := append([]time.Duration(nil), src.Delays...)
src.mu.Unlock()
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != time.Minute {
t.Errorf("step %+v, delays %v, want again in run 2 after 60s", s, delays)
}
}
// Puller.Prior over a steps state's read: the Run of the step the key holds, 0 for an absent key, a failed read or what is no
// step, and nil for no read.
func TestPriorRunsReadsTheRunOfTheStepAKeyHolds(t *testing.T) {