R4, R5, R6, R8: one take-in is one pipeline, its movers placed, ended and said per source revision

A take-in.taken with entries starts one pipeline keyed <branch>.<source> by compare-and-set, its
movers the entries; each mover's steps are placed from its own registration and entry, each step
and ask id naming its mover; a step end is matched to its mover's commit and fingerprint; the
pipeline passes when every mover's required steps passed, a required "nothing to run" counting
as passed. The movers get tiers from the delivery edges (kinds 2, 4, 5, 6; a cycle one tier,
said). pipeline-started and pipeline-ended are said on <branch>.<source>, the end once with per
mover its fingerprint, version, steps and tier, in tier order; waits on <branch>.<source>; step
events on <module>.<branch>.<commit>; no group-ended. Approval is one step per mover. A run on
request is one pipeline keyed <branch>.run.<id>, its source the branch's newest, members of one
branch only. A running pipeline is superseded no more. A composition step names the runner as
its asker. The seat's verb place is served, refusing every pipeline until the queue (R11, R14).
The seat list part takes no callers (H-A-66). Unit tests 77 to 80, integration 3.

Builds: design 47 H §746, §747 (run, start, a publish's key), §748 (a running pipeline runs to its end), §764 (group-ended, the key per module); lead answers H-A-3, H-A-35, H-A-51, H-A-52, H-A-53, H-A-66, H-A-70, H-A-72, H-A-83 (1), H-A-85, H-A-88

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent 10c806fe6f
commit a755925024
38 changed files with 1181 additions and 1014 deletions
@@ -37,7 +37,7 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
out := map[string][]seat.LogEntry{}
pkeys, err := h.Store.Keys("*.*.*")
pkeys, err := pipeline.AllKeys(h.Store)
if err != nil {
return nil, err
}
@@ -219,16 +219,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return placeHalf(ev, half)
}
placer.Store(&place)
// A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31,
// §350; A-182, A-202).
starter.Supersede = func(newer seat.Pipeline) {
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)))
if err != nil {
logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err)
} else if n > 0 {
logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n)
}
}
// The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock.
runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))
runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf)
@@ -281,7 +271,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go whileTheLogIsFull(ctx, fills, hearer, time.Minute)
// The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check
// their callers against (A-186): registered before the hub starts, read again once its events are live.
seats := seatlist.NewPart("runner", nil)
seats := seatlist.NewPart("runner")
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
@@ -161,9 +161,9 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
t.Fatal(err)
}
// A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module.
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m.1", Kind: "taken", Branch: "main", Source: 3,
taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: 3, Call: "0123456789abcdef",
Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
const key = "main.3.app"
const key = "main.3"
var p seat.Pipeline
waitFor(t, ctx, logs, func() bool {
// Heard before main's half is held, the pipeline starts with no steps, never placed from empty entries (A-336).
@@ -70,8 +70,8 @@ func hearPublished(ctx context.Context, hub handlers, starter *pipeline.Starter,
})
}
// hearTakenIn starts the pipelines of each take-in.taken heard (§44, §54): one per entry, the message acknowledged once
// they are written, and heard again when a write failed, so the keys already written start nothing twice.
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
var ev catalog.TakeInTaken
@@ -82,11 +82,11 @@ func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, log
}
n, err := starter.TakenIn(ev, v.View)
if err != nil {
logf("the take-in %s of %s: %d pipelines started, then %v; heard again", ev.Key, ev.Branch, n, err)
logf("the take-in of %s source %d: %d pipelines started, then %v; heard again", ev.Branch, ev.Source, n, err)
_ = m.Nak()
return
}
logf("the take-in %s of %s started %d pipelines", ev.Key, ev.Branch, n)
logf("the take-in of %s source %d started %d pipelines", ev.Branch, ev.Source, n)
_ = m.Ack()
})
}
@@ -233,7 +233,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
})
takeIn := func(module, commit, fp string, source, rev int64) {
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev,
taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: source, Revision: rev, Call: "0123456789abcdef",
Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
t.Fatal(err)
@@ -324,7 +324,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
const p3 = "main.3.app"
const p3 = "main.3"
takeIn("app", "c1", "src2:aa", 3, 1)
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
passBuild(p3, "app", "c1", "src2:aa")
@@ -394,8 +394,11 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
t.Errorf("the run %s at its end = %+v; want it ended failed", k, r)
}
}
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 {
t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e)
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || len(e.Movers) != 1 {
t.Fatalf("pipeline-ended = %+v; want failed, naming its one mover", e)
}
if m := read(p3).Movers[0]; m.Tests == nil || m.Tests.Unit != 5 || m.Tests.Integration != 2 {
t.Fatalf("the mover = %+v; want the tests counted once", m)
}
// An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the
@@ -467,7 +470,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
revision.Store(2)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
const p4, p5 = "main.4.app", "main.5.app"
const p4, p5 = "main.4", "main.5"
takeIn("app", "c1", "src2:aa", 4, 2)
placed(p4)
run4, r4 := runOfStep(p4, "app:program", "integration")
@@ -512,8 +515,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
// asked (§535, §647; A-470g, A-470h).
const web = "main.5.web"
takeIn("web", "w1", "src2:ww", 5, 2)
const web = "main.7"
takeIn("web", "w1", "src2:ww", 7, 2)
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
@@ -566,7 +569,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
}
revision.Store(3)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
const p6 = "main.6.app"
const p6 = "main.6"
takeIn("app", "c1", "src2:aa", 6, 3)
waitFor(t, ctx, logs, func() bool {
s := step(read(p6), "app:program", "unit")
@@ -47,6 +47,7 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }),
seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }),
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbPlace: verbOf(func(c string, in seat.PlaceInput) (any, error) { return v.Place(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
got = append(got, v)
}
slices.Sort(got)
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbPlace, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
slices.Sort(want)
if !slices.Equal(got, want) {
t.Errorf("the process answers %v, want %v", got, want)
@@ -64,13 +64,13 @@ func (listingCatalog) Subscribe(string, func(string, json.RawMessage) error) err
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
follow := func(silent bool) *seatlist.Part {
p := seatlist.NewPart("runner", nil)
p := seatlist.NewPart("runner")
if r := p.Follow(listingCatalog{silent: silent}, nil, t.Logf); r != nil {
<-r.Ready()
}
return p
}
if _, ok := seatlist.Definition(seatlist.NewPart("runner", nil).Reader, seatlist.Seat)(); ok {
if _, ok := seatlist.Definition(seatlist.NewPart("runner").Reader, seatlist.Seat)(); ok {
t.Error("a definition is answered before the seat list's reader started")
}
if _, ok := seatlist.Definition(follow(true).Reader, seatlist.Seat)(); ok {
@@ -6,7 +6,6 @@ import (
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Retention is one of the three ages the runner's own kept data are deleted at (design 47 G §330, *The runner's kept data*): its
@@ -226,16 +225,24 @@ func (a *Ager) deletePipeline(r Retention, key string, keepPassed bool) {
a.logf("aged: deleted the pipeline %s (%s)", key, r.Name)
}
// newest says whether p is its branch's newest pipeline for its module.
// newest says whether p is its branch's newest pipeline for one of its movers.
func (a *Ager) newest(p seat.Pipeline) (bool, error) {
pkeys, err := a.Pipelines.Keys(keys.EscapeName(p.Branch) + ".*." + p.Module)
pkeys, err := branchKeys(a.Pipelines, p.Branch)
if err != nil {
return false, fmt.Errorf("the pipelines of %s on %s are not listed: %w", p.Module, p.Branch, err)
return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err)
}
for _, k := range pkeys {
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source {
return false, nil
// Newest for one of its movers: no later pipeline of the branch moves it.
for _, m := range p.Movers {
newer := false
for _, k := range pkeys {
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) {
newer = true
break
}
}
if !newer {
return true, nil
}
}
return true, nil
return len(p.Movers) == 0, nil
}
@@ -76,11 +76,11 @@ func newAgedFixture() *agedFixture {
func sprintf(format string, a ...any) string { return fmt.Sprintf(format, a...) }
func (f *agedFixture) pipeline(branch string, source int64, outcome string, run string) string {
key := branch + "." + strconv.FormatInt(source, 10) + ".app"
key := branch + "." + strconv.FormatInt(source, 10)
f.pipes.memStore[key] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: key, Branch: branch, Source: source, Module: "app", Outcome: outcome, Run: run}, 1}
}{seat.Pipeline{Pipeline: key, Branch: branch, Source: source, Outcome: outcome, Run: run, Movers: []seat.Mover{{Module: "app"}}}, 1}
return key
}
@@ -29,40 +29,45 @@ func About(module, branch string) string {
// approvalDue is the index of the approval step to ask now (§26): waiting, not bypassed, every other step ended and every
// other required step passed or carried; -1 when none is due.
func approvalDue(p seat.Pipeline) int {
at := -1
for i, s := range p.Steps {
if s.Step == seat.StepResultStepApproval {
if s.Status != seat.StepResultStatusBypass && s.Outcome == seat.StepResultOutcomeWaiting {
at = i
}
func approvalDue(p seat.Pipeline) []int {
var out []int
for i, a := range p.Steps {
if a.Step != seat.StepResultStepApproval || a.Status == seat.StepResultStatusBypass || a.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
return -1
due := true
for _, s := range p.Steps {
if s.Module != a.Module || s.Step == seat.StepResultStepApproval {
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) || (s.Status == seat.StepResultStatusRequired && !stepPassed(s)) {
due = false
break
}
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
return -1
if due {
out = append(out, i)
}
}
return at
return out
}
// ApprovalAsk is the ask of a pipeline's approval step. Its id and its expiry follow from the pipeline and from when the
// step was asked, so the ask built again for the same pipeline is the same ask, digest included: a warrant is held to it.
func ApprovalAsk(p seat.Pipeline, from time.Time) (oc.Ask, error) {
func ApprovalAsk(p seat.Pipeline, module string, from time.Time) (oc.Ask, error) {
m, _ := MoverOf(p, module)
// An approval asked again (ask-again) is an ask of its own, its id following from when it was asked again, so the
// ask rebuilt from the step is the same and the earlier, decided ask is not its duplicate.
idOf := p.Pipeline + "\n" + p.Commit
idOf := p.Pipeline + "\n" + m.Module + "\n" + m.Commit
for _, st := range p.Steps {
if st.Step == seat.StepResultStepApproval && st.Rerun {
if st.Step == seat.StepResultStepApproval && st.Module == module && st.Rerun {
idOf += "\nasked again\n" + from.UTC().Format(time.RFC3339Nano)
}
}
sum := sha256.Sum256([]byte(idOf))
act := func(do string) (string, error) {
return operatorchannelclient.ActDigest(operatorchannelclient.Act{"pipeline": p.Pipeline, "commit": p.Commit,
"fingerprint": p.Fingerprint, "do": do})
return operatorchannelclient.ActDigest(operatorchannelclient.Act{"pipeline": p.Pipeline, "module": m.Module, "commit": m.Commit,
"fingerprint": m.Fingerprint, "do": do})
}
approve, err := act(OptionApprove)
if err != nil {
@@ -72,14 +77,14 @@ func ApprovalAsk(p seat.Pipeline, from time.Time) (oc.Ask, error) {
if err != nil {
return oc.Ask{}, err
}
a := oc.Ask{ID: "approval-" + hex.EncodeToString(sum[:12]), About: About(p.Module, p.Branch), Who: operatorchannelclient.Operator,
Headline: trimRunes("Deliver "+p.Module+" from "+p.Branch+"?", operatorchannelclient.HeadlineLength),
Explanation: fmt.Sprintf("Every required step of %s's pipeline at commit %s on %s passed; its policy asks your approval before it is delivered.", p.Module, p.Commit, p.Branch),
Details: "pipeline " + p.Pipeline + ", fingerprint " + p.Fingerprint,
a := oc.Ask{ID: "approval-" + hex.EncodeToString(sum[:12]), About: About(m.Module, p.Branch), Who: operatorchannelclient.Operator,
Headline: trimRunes("Deliver "+m.Module+" from "+p.Branch+"?", operatorchannelclient.HeadlineLength),
Explanation: fmt.Sprintf("Every required step of %s's pipeline at commit %s on %s passed; its policy asks your approval before it is delivered.", m.Module, m.Commit, p.Branch),
Details: "pipeline " + p.Pipeline + ", fingerprint " + m.Fingerprint,
Expires: operatorchannelclient.FormatTime(from.Add(operatorchannelclient.ApproveLasts)),
OnExpiry: "the step ends declined and the commit is not delivered; ask-again asks once more",
Options: []oc.Option{
{ID: OptionApprove, Label: "Approve", Does: "delivers this commit of " + p.Module, Level: operatorchannelclient.Approve, Binds: approve},
{ID: OptionApprove, Label: "Approve", Does: "delivers this commit of " + m.Module, Level: operatorchannelclient.Approve, Binds: approve},
{ID: OptionDecline, Label: "Decline", Does: "ends the step declined; nothing is delivered", Level: operatorchannelclient.Approve, Binds: decline},
}}
return a, operatorchannelclient.Check(a, from)
@@ -101,26 +106,29 @@ type Asker interface {
// askApproval asks p's approval step when it is due, before the pipeline is written: an ask published and then not
// written is published again, the same ask, when its step's end is heard again.
func askApproval(key string, p *seat.Pipeline, from time.Time, asker Asker) (bool, error) {
i := approvalDue(*p)
if i < 0 || asker == nil {
return false, nil
func askApproval(key string, p *seat.Pipeline, from time.Time, asker Asker) ([]int, error) {
if asker == nil {
return nil, nil
}
// An approval asked again keeps the time ask-again wrote, so an ask published and then not written is the same ask.
if s := p.Steps[i]; s.Rerun && s.Started != nil {
from = *s.Started
var asked []int
for _, i := range approvalDue(*p) {
at := from
if s := p.Steps[i]; s.Rerun && s.Started != nil {
at = *s.Started
}
a, err := ApprovalAsk(*p, p.Steps[i].Module, at)
if err != nil {
return asked, fmt.Errorf("the approval of %s's %s cannot be asked: %w", key, p.Steps[i].Module, err)
}
if err := asker.Ask(a); err != nil {
return asked, fmt.Errorf("the approval of %s's %s is not asked: %w", key, p.Steps[i].Module, err)
}
s := &p.Steps[i]
s.Outcome, s.Ask, s.Seat, s.Started = seat.StepResultOutcomeRunning, a.ID, seat.Seat, &at
s.Why = "waiting on the operator"
asked = append(asked, i)
}
a, err := ApprovalAsk(*p, from)
if err != nil {
return false, fmt.Errorf("the approval of %s cannot be asked: %w", key, err)
}
if err := asker.Ask(a); err != nil {
return false, fmt.Errorf("the approval of %s is not asked: %w", key, err)
}
s := &p.Steps[i]
s.Outcome, s.Ask, s.Seat, s.Started = seat.StepResultOutcomeRunning, a.ID, seat.Seat, &from
s.Why = "waiting on the operator"
return true, nil
return asked, nil
}
// ErrNoAsk is a warrant naming no approval step the runner keeps for its about.
@@ -140,7 +148,7 @@ func decide(p *seat.Pipeline, w oc.Warrant, asker string, now time.Time) (bool,
}
switch w.Outcome {
case oc.WarrantOutcomeChosen:
asked, err := ApprovalAsk(*p, *s.Started)
asked, err := ApprovalAsk(*p, s.Module, *s.Started)
if err != nil {
return false, err
}
@@ -188,11 +196,12 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
return seat.Pipeline{}, fmt.Errorf("%w: %s", ErrNoRecord, event.Ask)
}
w := *rec
module, branch, ok := strings.Cut(w.About, ".")
if !ok {
_, escaped, ok := strings.Cut(w.About, ".")
branch, err := keys.UnescapeName(escaped)
if !ok || err != nil {
return seat.Pipeline{}, fmt.Errorf("%w: the warrant for %s is about %q, not a module and a branch", ErrNoAsk, w.Ask, w.About)
}
found, err := h.Store.Keys(branch + ".*." + module)
found, err := branchKeys(h.Store, branch)
if err != nil {
return seat.Pipeline{}, err
}
@@ -210,7 +219,7 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
if err != nil || !changed {
return p, err
}
ended := endPipeline(&p, now)
ended := h.endPipeline(&p, now)
i := stepOfAsk(p, w.Ask)
if i >= 0 {
p.Unlogged = append(p.Unlogged, endEntry(key, p.Steps[i], now))
@@ -226,7 +235,7 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
}
}
h.appendUnlogged(key)
return p, h.said(p, false, ended)
return p, h.said(p, nil, ended)
}
}
return seat.Pipeline{}, fmt.Errorf("%w: %s about %s", ErrNoAsk, w.Ask, w.About)
@@ -8,53 +8,93 @@ import (
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
)
// pipeline-ended carries the module's test counts and says it over budget past 500 unit or 50 integration tests (design 47 G
// §24, §419), and a group's one pipeline ends with group-ended under the group once every member ended (§73, §332).
func TestPipelineEndedCarriesTestCountsAndAGroupEndsOnItsMembers(t *testing.T) {
// twoMovers is the fixture's pipeline moving a provider lib (tier 1) and its consumer app (tier 2), each with one required
// lint step asked of node-build-go.
func twoMovers(t *testing.T) (*Hearer, memStore, map[string]any, *[]string) {
t.Helper()
h, store, _, _, _, _ := fixture()
bodies := map[string]any{}
var order []string
order := &[]string{}
h.Say = func(subject string, body any) error {
ev := subject[strings.Index(subject, ".event.")+7:]
bodies[ev] = body
order = append(order, ev)
*order = append(*order, ev)
return nil
}
e := store[pkey]
e.p.Steps = e.p.Steps[:2]
e.p.Steps[1].Status = seat.StepResultStatusRequired
e.p.Group = "g1"
e.p.Movers = []seat.Mover{{Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Tier: 1, Version: "1.2.0"},
{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Tier: 2}}
e.p.Steps = []seat.StepResult{
{Module: "app", Artifact: "app:program", Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "a1", Seat: nodebuildgo.Seat},
{Module: "lib", Artifact: "lib:lib", Step: seat.StepResultStepUnit, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "b1", Seat: nodebuildgo.Seat}}
store[pkey] = e
// A second member of the same group, on the same source revision, still running.
store["main.5.lib"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{V: 1, Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Commit: "c2", Group: "g1", Outcome: seat.PipelineOutcomeRunning,
Steps: []seat.StepResult{{Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "b1", Seat: nodebuildgo.Seat}}}, 1}
return h, store, bodies, order
}
func libEnded(outcome string) StepEnded {
ev := ended("b1", "unit", outcome)
ev.Module, ev.Commit, ev.Fingerprint = "lib", "c2", "src2:bb"
return ev
}
// R8/AC1 (design 47 H §746, acceptance test 112): a pipeline of a provider and its consumer says one pipeline-ended on
// <branch>.<source>, naming both movers in tier order, each with its fingerprint, version, steps and tier; R8/AC2 (§764): no
// group-ended is said. Each mover's test counts are kept in its value, over budget past 500 unit tests (G §24, §419).
func TestOnePipelineEndedNamesEveryMoverWithItsTier(t *testing.T) {
h, store, bodies, order := twoMovers(t)
lint := ended("a1", "lint", "passed")
unit := ended("a2", "unit", "passed")
unit.Tests = &nodebuildgo.TestCounts{Unit: 501, Integration: 0}
unit := libEnded("passed")
unit.Tests = &nodebuildgo.TestCounts{Unit: 501}
for _, ev := range []StepEnded{lint, unit} {
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ev); err != nil {
t.Fatal(err)
}
}
pe, ok := bodies["pipeline-ended.app.main.g1"].(seat.PipelineEnded)
if !ok || pe.Tests == nil || pe.Tests.Unit != 501 || !pe.OverBudget {
t.Fatalf("pipeline-ended = %+v (said %v)", bodies["pipeline-ended.app.main.g1"], order)
pe, ok := bodies["pipeline-ended.main.5"].(seat.PipelineEnded)
if !ok || pe.Outcome != seat.PipelineEndedOutcomePassed || pe.Pipeline != pkey || pe.Source != 5 || len(pe.Movers) != 2 {
t.Fatalf("pipeline-ended = %+v (said %v)", bodies["pipeline-ended.main.5"], *order)
}
if _, said := bodies["group-ended.main.g1"]; said {
t.Fatalf("group-ended said while a member runs: %v", order)
lib, app := pe.Movers[0], pe.Movers[1]
if lib.Module != "lib" || lib.Tier != 1 || lib.Fingerprint != "src2:bb" || lib.Version != "1.2.0" || len(lib.Steps) != 1 ||
app.Module != "app" || app.Tier != 2 || app.Fingerprint != "src2:aa" || app.Version != "" || len(app.Steps) != 1 {
t.Errorf("movers = %+v, want lib tier 1 at 1.2.0 then app tier 2, each with its one step", pe.Movers)
}
lib := ended("b1", "lint", "passed")
lib.Pipeline, lib.Module, lib.Commit, lib.Fingerprint = "main.5.lib", "lib", "c2", ""
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", lib); err != nil {
t.Fatal(err)
ends := 0
for _, ev := range *order {
if strings.HasPrefix(ev, "pipeline-ended.") {
ends++
}
if strings.HasPrefix(ev, "group-ended") {
t.Errorf("said %s: the runner says no group-ended (§764)", ev)
}
}
ge, ok := bodies["group-ended.main.g1"].(seat.GroupEnded)
if !ok || ge.Outcome != seat.GroupEndedOutcomePassed || ge.Group != "g1" || ge.Branch != "main" || ge.Source != 5 || len(ge.Pipelines) != 2 {
t.Fatalf("group-ended = %+v (said %v)", bodies["group-ended.main.g1"], order)
if ends != 1 {
t.Errorf("pipeline-ended said %d times, want once: %v", ends, *order)
}
kept := store[pkey].p.Movers
if kept[0].Tests == nil || kept[0].Tests.Unit != 501 || !kept[0].OverBudget || kept[1].OverBudget {
t.Errorf("kept movers = %+v, want lib's 501 unit tests over budget", kept)
}
}
// R6/AC1 and AC2 (design 47 H §746; ADR 0682 §8, H-A-3): a mover's required step that ends `nothing to run` counts as
// passed, so the pipeline passes; one mover's required step failed fails the pipeline, naming that mover's step.
func TestThePipelinePassesOnNothingToRunAndFailsOnOneMover(t *testing.T) {
for _, c := range []struct {
name, lib, want string
}{
{"nothing to run", seat.StepResultOutcomeNothingToRun, seat.PipelineOutcomePassed},
{"one mover failed", seat.StepResultOutcomeFailed, seat.PipelineOutcomeFailed},
} {
h, store, _, _ := twoMovers(t)
for _, ev := range []StepEnded{ended("a1", "lint", "passed"), libEnded(c.lib)} {
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ev); err != nil {
t.Fatal(err)
}
}
p := store[pkey].p
if p.Outcome != c.want || (c.want == seat.PipelineOutcomeFailed && !strings.Contains(p.Why, "lib lib:lib unit failed")) {
t.Errorf("%s: the pipeline ended %s (%s), want %s", c.name, p.Outcome, p.Why, c.want)
}
}
// A pipeline of no group says no group-ended.
}
@@ -10,7 +10,6 @@ import (
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// PipelineFailedKind is the kind of the runner's finding of a failed head (design 47 G §33, §540).
@@ -49,10 +48,10 @@ func (f *Failures) logf(format string, args ...any) {
// failedSteps are the required steps of p that failed or could not run, the approval excepted: an approval's end is the
// operator's, not a failure of the commit.
func failedSteps(p seat.Pipeline) []seat.StepResult {
func failedSteps(p seat.Pipeline, module string) []seat.StepResult {
var out []seat.StepResult
for _, s := range p.Steps {
if s.Status == seat.StepResultStatusRequired && s.Step != seat.StepResultStepApproval &&
if s.Module == module && s.Status == seat.StepResultStatusRequired && s.Step != seat.StepResultStepApproval &&
(s.Outcome == seat.StepResultOutcomeFailed || s.Outcome == seat.StepResultOutcomeCouldNotRun) {
out = append(out, s)
}
@@ -74,22 +73,22 @@ func (f *Failures) held(commit string) *condseat.Finding {
// a step of a commit already found failed that is not one writes its owner when it had none, and a passing required step clears
// the findings of the older revisions it makes good.
func (f *Failures) StepEnded(store Updater, p seat.Pipeline, s seat.StepResult) {
if f == nil || f.Findings == nil || p.Commit == "" {
m, ok := MoverOf(p, s.Module)
if f == nil || f.Findings == nil || !ok || m.Commit == "" {
return
}
ctx := context.Background()
switch {
case len(failedSteps(p)) > 0:
f.write(ctx, p)
case len(failedSteps(p, m.Module)) > 0:
f.write(ctx, p, m)
case s.Outcome == seat.StepResultOutcomePassed || s.Outcome == seat.StepResultOutcomeCarried:
f.clearNewer(ctx, store, p)
f.clearNewer(ctx, store, p, m)
}
}
// write writes p's commit's finding; the owner it held stays, and one it lacked is read again here, so the owner of a
// failed read is written at the next step end of that commit (A-191).
func (f *Failures) write(ctx context.Context, p seat.Pipeline) {
steps := failedSteps(p)
// write raises or rewrites the mover m's finding (§33).
func (f *Failures) write(ctx context.Context, p seat.Pipeline, m seat.Mover) {
steps := failedSteps(p, m.Module)
var names []string
for _, s := range steps {
names = append(names, strings.TrimSpace(s.Artifact+" "+s.Step)+" ("+s.Outcome+")")
@@ -99,64 +98,64 @@ func (f *Failures) write(ctx context.Context, p seat.Pipeline) {
now = f.Now().UTC()
}
owner, why := "", ""
if old := f.held(p.Commit); old != nil {
if old := f.held(m.Commit); old != nil {
owner = old.Owner
}
if owner == "" {
owner, why = f.author(p)
owner, why = f.author(m)
}
explanation := fmt.Sprintf("The required steps %s of %s did not pass on the branch %s at the commit %s. A newer commit of the branch that passes them clears this.",
strings.Join(names, ", "), p.Module, p.Branch, p.Commit)
strings.Join(names, ", "), m.Module, p.Branch, m.Commit)
if why != "" {
explanation += " " + why
}
err := f.Findings.Write(ctx, condseat.Finding{Scope: mcc.ScopeMerge, ID: p.Commit, Kind: PipelineFailedKind, Severity: condseat.FindingSeverityWarning,
Summary: fmt.Sprintf("%s on %s at %s: %s", p.Module, p.Branch, p.Commit, strings.Join(names, ", ")),
Headline: fmt.Sprintf("A required step of %s failed on %s", p.Module, p.Branch),
err := f.Findings.Write(ctx, condseat.Finding{Scope: mcc.ScopeMerge, ID: m.Commit, Kind: PipelineFailedKind, Severity: condseat.FindingSeverityWarning,
Summary: fmt.Sprintf("%s on %s at %s: %s", m.Module, p.Branch, m.Commit, strings.Join(names, ", ")),
Headline: fmt.Sprintf("A required step of %s failed on %s", m.Module, p.Branch),
Explanation: explanation, Needs: "a newer commit of " + p.Branch + " that passes " + strings.Join(names, ", "),
Owner: owner, Source: "pipeline-runner." + PipelineFailedKind, Observer: f.Observer, At: now,
Subject: condseat.Subject{Scope: mcc.ScopeMerge, ID: p.Commit}})
Subject: condseat.Subject{Scope: mcc.ScopeMerge, ID: m.Commit}})
if err != nil {
f.logf("the finding %s of %s at %s is not written: %v", PipelineFailedKind, p.Module, p.Commit, err)
f.logf("the finding %s of %s at %s is not written: %v", PipelineFailedKind, m.Module, m.Commit, err)
}
}
// author is the finding's owner `forge:<account>` and, when it has none, why it has none (A-191).
func (f *Failures) author(p seat.Pipeline) (string, string) {
// author is the forge account of the mover's commit's author, read through the forge (§33).
func (f *Failures) author(m seat.Mover) (string, string) {
if f.Author == nil {
return "", "The commit's author is not read, so the finding has no owner."
}
account, err := f.Author(p.Repository, p.Commit)
account, err := f.Author(m.Repository, m.Commit)
if err != nil {
f.logf("the author of %s in %s is not read, so its finding %s has no owner: %v", p.Commit, p.Repository, PipelineFailedKind, err)
f.logf("the author of %s in %s is not read, so its finding %s has no owner: %v", m.Commit, m.Repository, PipelineFailedKind, err)
return "", "The commit's author is not known (" + err.Error() + "), so the finding has no owner."
}
if !forgeOwner.MatchString("forge:" + account) {
f.logf("the author %q of %s is no forge account name, so its finding %s has no owner", account, p.Commit, PipelineFailedKind)
f.logf("the author %q of %s is no forge account name, so its finding %s has no owner", account, m.Commit, PipelineFailedKind)
return "", "The commit's author is not known as a forge account, so the finding has no owner."
}
return "forge:" + account, ""
}
// clearNewer clears the finding of each older commit of p's module and branch whose every failed step a revision newer than it
// has passed (§33: cleared when a newer revision of that branch passes that step).
func (f *Failures) clearNewer(ctx context.Context, store Updater, p seat.Pipeline) {
pkeys, err := store.Keys(keys.EscapeName(p.Branch) + ".*." + p.Module)
// clearNewer clears the finding of each older pipeline of the branch moving m's module whose failed steps a newer one passed.
func (f *Failures) clearNewer(ctx context.Context, store Updater, p seat.Pipeline, m seat.Mover) {
pkeys, err := branchKeys(store, p.Branch)
if err != nil {
f.logf("the pipelines of %s on %s are not listed, so no finding %s is cleared: %v", p.Module, p.Branch, PipelineFailedKind, err)
f.logf("the pipelines of %s on %s are not listed, so no finding %s is cleared: %v", m.Module, p.Branch, PipelineFailedKind, err)
return
}
var all []seat.Pipeline
for _, k := range pkeys {
if q, _, err := store.Get(k); err == nil {
if q, _, err := store.Get(k); err == nil && Moves(q, m.Module) {
all = append(all, q)
}
}
for _, old := range all {
if old.Source >= p.Source || f.held(old.Commit) == nil {
was, _ := MoverOf(old, m.Module)
if old.Source >= p.Source || f.held(was.Commit) == nil {
continue
}
failed := failedSteps(old)
failed := failedSteps(old, m.Module)
if len(failed) == 0 {
continue
}
@@ -165,21 +164,20 @@ func (f *Failures) clearNewer(ctx context.Context, store Updater, p seat.Pipelin
made = made && passedAfter(all, old, s)
}
if made {
if err := f.Findings.Clear(ctx, mcc.ScopeMerge, old.Commit, PipelineFailedKind, "a newer revision of "+p.Branch+" passes its steps"); err != nil {
f.logf("the finding %s of %s at %s is not cleared: %v", PipelineFailedKind, p.Module, old.Commit, err)
if err := f.Findings.Clear(ctx, mcc.ScopeMerge, was.Commit, PipelineFailedKind, "a newer revision of "+p.Branch+" passes its steps"); err != nil {
f.logf("the finding %s of %s at %s is not cleared: %v", PipelineFailedKind, m.Module, was.Commit, err)
}
}
}
}
// passedAfter says whether a pipeline newer than old passed the step s of old.
func passedAfter(all []seat.Pipeline, old seat.Pipeline, s seat.StepResult) bool {
for _, q := range all {
if q.Source <= old.Source {
continue
}
for _, t := range q.Steps {
if t.Artifact == s.Artifact && t.Step == s.Step && (t.Outcome == seat.StepResultOutcomePassed || t.Outcome == seat.StepResultOutcomeCarried) {
if t.Module == s.Module && t.Artifact == s.Artifact && t.Step == s.Step && (t.Outcome == seat.StepResultOutcomePassed || t.Outcome == seat.StepResultOutcomeCarried) {
return true
}
}
@@ -59,7 +59,7 @@ func TestAFailedRequiredStepIsTheFindingOfItsCommit(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
h, store, _, _, _, _ := fixture()
p := store[pkey]
p.p.Repository = "novox/app"
p.p.Movers[0].Repository = "novox/app"
store[pkey] = p
f := newHeld()
h.Failures = &Failures{Findings: f, Observer: "mesh-delivery-pipeline", Now: h.Now,
@@ -134,13 +134,13 @@ func TestANewerPassingRevisionClearsTheFinding(t *testing.T) {
t.Fatal(err)
}
newer := func(source int64, commit, ask string) string {
key := "main." + strconv.FormatInt(source, 10) + ".app"
key := "main." + strconv.FormatInt(source, 10)
store[key] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: commit, Fingerprint: "src2:aa",
Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{{Artifact: "program", Step: seat.StepResultStepLint,
Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: ask, Seat: nodebuildgo.Seat}}}, 1}
}{seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: source,
Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{{Module: "app", Artifact: "program", Step: seat.StepResultStepLint,
Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: ask, Seat: nodebuildgo.Seat}}, Movers: []seat.Mover{{Module: "app", Commit: commit, Fingerprint: "src2:aa"}}}, 1}
return key
}
// A newer head that fails the step too is a finding of its own, and the older one stays.
@@ -34,7 +34,7 @@ func TestAStoreFillingIsSaidAndAFullStoreRefusesAStart(t *testing.T) {
}
bytes = 100
s := &Starter{Pipelines: failingCreate{}, Fills: f, Say: func(string, any) error { return nil }}
ok, err := s.Start(Begin{Branch: "main", Source: 1, Module: "app"})
ok, err := s.Start(Begin{Branch: "main", Source: 1, Movers: []seat.Mover{{Module: "app"}}})
if ok || !errors.Is(err, ErrFull) {
t.Fatalf("a start with pipelines full: %v, %v; want it refused with ErrFull", ok, err)
}
@@ -42,7 +42,7 @@ func TestAStoreFillingIsSaidAndAFullStoreRefusesAStart(t *testing.T) {
t.Error("the refused start is not said in the journal")
}
bytes = 10
if ok, err := s.Start(Begin{Branch: "main", Source: 1, Module: "app"}); !ok || err != nil {
if ok, err := s.Start(Begin{Branch: "main", Source: 1, Movers: []seat.Mover{{Module: "app"}}}); !ok || err != nil {
t.Fatalf("a start on room: %v, %v", ok, err)
}
}
@@ -0,0 +1,132 @@
package pipeline
import (
"slices"
"sort"
"strings"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// tierKinds are the catalog graph's edge kinds that order delivery (ADR 0666 §14; design 47 H H-A-51): a fetched artifact
// (2), a provider before its consumer (4), a part that acted for another (5) and a served verb (6). Kinds 1 and 3 order
// the build, not the delivery.
var tierKinds = map[int64]bool{2: true, 4: true, 5: true, 6: true}
// moduleOf is the module of an artifact reference, `<module>:<artifact>`.
func moduleOf(ref string) string {
m, _, _ := strings.Cut(ref, ":")
return m
}
// Tiers gives each mover its tier, its place in the movers' delivery order (design 47 H §746, H-A-51): the movers an edge
// of a delivery kind from another mover points to come after it, acting parts and providers first; movers no edge orders
// share a tier; the movers of a cycle share one tier, said in the journal. Tiers count from 1. The movers are answered in
// that order, by tier and then by module.
func Tiers(movers []seat.Mover, entries map[string]catalog.CatalogEntry, logf func(string, ...any)) []seat.Mover {
in := map[string]bool{}
for _, m := range movers {
in[m.Module] = true
}
// after[a] are the movers a must come after.
after := map[string][]string{}
for _, m := range movers {
for _, x := range entries[m.Module].Edges {
from := moduleOf(x.From)
if tierKinds[x.Kind] && in[from] && from != m.Module && !slices.Contains(after[m.Module], from) {
after[m.Module] = append(after[m.Module], from)
}
}
}
comp := components(movers, after)
// The tier of a component is one above the highest tier of a component it comes after.
tier := map[int]int64{}
var tierOf func(c int, seen map[int]bool) int64
tierOf = func(c int, seen map[int]bool) int64 {
if t, ok := tier[c]; ok {
return t
}
seen[c] = true
t := int64(1)
for _, m := range movers {
if comp[m.Module] != c {
continue
}
for _, d := range after[m.Module] {
if dc := comp[d]; dc != c && !seen[dc] {
t = max(t, tierOf(dc, seen)+1)
}
}
}
tier[c] = t
return t
}
members := map[int][]string{}
for _, m := range movers {
members[comp[m.Module]] = append(members[comp[m.Module]], m.Module)
}
for _, cycle := range members {
if len(cycle) > 1 && logf != nil {
sort.Strings(cycle)
logf("the movers %s make a cycle of delivery edges: they share one tier (H-A-51)", strings.Join(cycle, ", "))
}
}
out := append([]seat.Mover(nil), movers...)
for i := range out {
out[i].Tier = tierOf(comp[out[i].Module], map[int]bool{})
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Tier != out[j].Tier {
return out[i].Tier < out[j].Tier
}
return out[i].Module < out[j].Module
})
return out
}
// components numbers the strongly connected components of the movers under after (Tarjan), so a cycle is one component.
func components(movers []seat.Mover, after map[string][]string) map[string]int {
index, low, comp := map[string]int{}, map[string]int{}, map[string]int{}
var stack []string
on := map[string]bool{}
next, count := 0, 0
var visit func(v string)
visit = func(v string) {
index[v], low[v] = next, next
next++
stack = append(stack, v)
on[v] = true
for _, w := range after[v] {
if _, seen := index[w]; !seen {
visit(w)
low[v] = min(low[v], low[w])
} else if on[w] {
low[v] = min(low[v], index[w])
}
}
if low[v] == index[v] {
for {
w := stack[len(stack)-1]
stack = stack[:len(stack)-1]
on[w] = false
comp[w] = count
if w == v {
break
}
}
count++
}
}
names := make([]string, 0, len(movers))
for _, m := range movers {
names = append(names, m.Module)
}
sort.Strings(names)
for _, n := range names {
if _, seen := index[n]; !seen {
visit(n)
}
}
return comp
}
@@ -0,0 +1,44 @@
package pipeline
import (
"strings"
"testing"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// R8/AC1 (design 47 H §746, H-A-51): the movers' delivery order as tiers: a provider before its consumer (kind 4), an acting
// part first (kind 5); a package edge (kind 1) orders the build, not the tier; movers no edge orders share a tier; the
// movers of a cycle share one tier, said in the journal.
func TestTiersOrderTheMoversDelivery(t *testing.T) {
edge := func(from string, kind int64) catalog.Edge {
return catalog.Edge{From: from + ":x", To: "y", Kind: kind}
}
entries := map[string]catalog.CatalogEntry{
"store": {Edges: []catalog.Edge{edge("builder", 5)}},
"app": {Edges: []catalog.Edge{edge("store", 4), edge("mesh-sdk", 1)}},
"loner": {Edges: []catalog.Edge{edge("elsewhere", 4)}},
"ping": {Edges: []catalog.Edge{edge("pong", 6)}},
"pong": {Edges: []catalog.Edge{edge("ping", 6)}},
"builder": {},
}
var said []string
movers := []seat.Mover{{Module: "app"}, {Module: "store"}, {Module: "builder"}, {Module: "mesh-sdk"}, {Module: "loner"}, {Module: "ping"}, {Module: "pong"}}
got := Tiers(movers, entries, func(f string, a ...any) { said = append(said, f) })
tier := map[string]int64{}
var order []string
for _, m := range got {
tier[m.Module] = m.Tier
order = append(order, m.Module)
}
want := map[string]int64{"builder": 1, "mesh-sdk": 1, "loner": 1, "ping": 1, "pong": 1, "store": 2, "app": 3}
for m, w := range want {
if tier[m] != w {
t.Errorf("%s is tier %d, want %d (order %v)", m, tier[m], w, order)
}
}
if order[len(order)-1] != "app" || len(said) != 1 || !strings.Contains(said[0], "cycle") {
t.Errorf("order %v, said %v; want app last and the cycle said once", order, said)
}
}
@@ -1,25 +1,25 @@
package pipeline
import (
"strings"
"testing"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// R3/AC1 (design 47 H §749, §763): a pipeline's pins are the indexer's. The runner says no pinned and no freed: not at a
// pipeline's start, its end, its supersession or a rerun.
// pipeline's start or its end.
func TestTheRunnerSaysNoPin(t *testing.T) {
var pins []string
say := func(subject string, body any) error {
if _, ok := body.(delivery.Pins); ok || subject == delivery.SubjectPinned || subject == delivery.SubjectFreed {
if strings.Contains(subject, ".pinned") || strings.Contains(subject, ".freed") {
pins = append(pins, subject)
}
return nil
}
s := &Starter{Pipelines: created{}, Say: say}
if ok, err := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); !ok || err != nil {
if ok, err := s.Start(Begin{Branch: "main", Source: 7, Revision: 12, Movers: []seat.Mover{{Module: "a", Commit: "c1"}}}); !ok || err != nil {
t.Fatalf("start: %v %v", ok, err)
}
h, store, _, _, _, _ := fixture()
@@ -27,12 +27,9 @@ func TestTheRunnerSaysNoPin(t *testing.T) {
p := store[pkey]
p.p.Revision = 12
store[pkey] = p
if err := h.said(store[pkey].p, false, true); err != nil {
if err := h.said(store[pkey].p, nil, true); err != nil {
t.Fatal(err)
}
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
t.Fatalf("supersede %d (%v)", n, err)
}
if len(pins) != 0 {
t.Errorf("the runner said %v, want no pinned and no freed", pins)
}
@@ -40,7 +37,7 @@ func TestTheRunnerSaysNoPin(t *testing.T) {
// A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view
// holds; while the view holds an older one it starts with no step and places none, and the view reaching the revision places
// them. An event that names no revision starts at the view's, as before.
// them.
func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
@@ -49,12 +46,12 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
viewRevision := int64(356)
view := func(string) (Half, bool) { return Half{Revision: viewRevision, Entries: entries}, true }
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357,
taken := catalog.TakeInTaken{Branch: "main", Source: 947, Revision: 357,
Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the take-in started %d (%v)", n, err)
}
p, _, _ := store.Get("main.947.app")
p, _, _ := store.Get("main.947")
if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 {
t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks))
}
@@ -63,18 +60,10 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
}
viewRevision = 357
n, err := s.Waiting(store.memStore, "main", view)
p, _, _ = store.Get("main.947.app")
p, _, _ = store.Get("main.947")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 {
t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision)
}
// An event of before A-448 names no revision: the view's.
taken = catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 948, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the old take-in started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.948.app"); p.Revision != 357 || len(p.Steps) == 0 {
t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p)
}
}
// A-448: the view is read before the starts write; a view that reaches the revision meanwhile places the held-back pipelines at
@@ -94,11 +83,11 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t
}
return Half{Revision: 356, Entries: entries}, true
}
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
taken := catalog.TakeInTaken{Branch: "main", Source: 947, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.947.app"); len(p.Steps) == 0 || len(asks.asks) == 0 {
if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 {
t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks))
}
}
@@ -125,7 +114,7 @@ func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) {
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
view := func(string) (Half, bool) { return Half{Revision: 357, Entries: entries}, true }
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 1, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, view); n != 1 || err != nil {
if n, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Source: 1, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, view); n != 1 || err != nil {
t.Fatalf("started %d (%v)", n, err)
}
before := len(asks.asks)
@@ -54,11 +54,11 @@ var buildSteps = []string{seat.StepResultStepLint, seat.StepResultStepUnit, seat
var moduleSteps = []string{seat.StepResultStepComposition, seat.StepResultStepReview, seat.StepResultStepApproval}
// AskID is a step's ask id in the usual ask id form the holders accept (buildstep.CheckAskID): base 36 of started in
// milliseconds, then six hex digits of the sha256 of the pipeline, the artifact and the step, so the same placement makes
// the same id at every start and the bus dedupes on it; started is the pipeline's start time, or a rerun's own (design 47
// G A-297).
func AskID(started time.Time, pipeline, artifact, step string) string {
sum := sha256.Sum256([]byte(pipeline + "\n" + artifact + "\n" + step))
// milliseconds, then six hex digits of the sha256 of the pipeline, the mover, the artifact and the step, so the same
// placement makes the same id at every start and the bus dedupes on it, and each mover's step its own id (design 47 H
// §746); started is the pipeline's start time, or a rerun's own (design 47 G A-297).
func AskID(started time.Time, pipeline, module, artifact, step string) string {
sum := sha256.Sum256([]byte(pipeline + "\n" + module + "\n" + artifact + "\n" + step))
return strconv.FormatInt(started.UnixMilli(), 36) + hex.EncodeToString(sum[:3])
}
@@ -76,16 +76,16 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
// asked of the build seat the artifact's language names in the catalog half (§323), or, where no seat names it, ended
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
// work queues, one per running step.
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
return place(p, reg, entries, now, asker, true)
func Place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
return place(p, m, reg, entries, now, asker, true)
}
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
// naming the run's ask and the step that asked it (A-469m, A-469q).
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
entry := entries[p.Module]
func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
entry := entries[m.Module]
candidate, named := CandidateOf(p)
share = share && named
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
@@ -93,13 +93,13 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
var steps []seat.StepResult
var asks []Asked
// The build step is every pipeline's first, required whatever the policy names, asked of the seat mesh-builds for the
// module's own entry only (A-99, A-97): a group's other members are built by their own pipelines.
build := seat.StepResult{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Fingerprint: p.Fingerprint,
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(startOf(p, now), p.Pipeline, "", seat.StepResultStepBuild)}
// The build step is every mover's first, required whatever the policy names, asked of the seat mesh-builds for the
// mover's own entry only (A-99, A-97): each mover is built by its own step (§746).
build := seat.StepResult{Step: seat.StepResultStepBuild, Module: m.Module, Status: seat.StepResultStatusRequired, Fingerprint: m.Fingerprint,
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(startOf(p, now), p.Pipeline, m.Module, "", seat.StepResultStepBuild)}
steps = append(steps, build)
asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch,
Revision: p.Revision, Source: p.Source, Entries: []builds.BuildEntry{{Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint}}}})
asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Commit: m.Commit, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch,
Revision: p.Revision, Source: p.Source, Entries: []builds.BuildEntry{{Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint}}}})
for _, a := range arts {
name := a.Ref
if _, after, ok := strings.Cut(a.Ref, ":"); ok {
@@ -111,7 +111,7 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
status, why = seat.StepResultStatusBypass, fmt.Sprintf("an artifact of kind %s has no %s step: the builder packs or fetches it", a.Kind, step)
}
s := seat.StepResult{Step: step, Artifact: a.Ref, Status: status, Fingerprint: p.Fingerprint}
s := seat.StepResult{Step: step, Module: m.Module, Artifact: a.Ref, Status: status, Fingerprint: m.Fingerprint}
if status == seat.StepResultStatusBypass {
s.Outcome, s.Why = seat.StepResultOutcomeBypassed, why
steps = append(steps, s)
@@ -135,11 +135,11 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
continue
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
body := stepAsk(p, entry, a, language, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language}
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step)
body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body, Toolchain: a.Language, Commit: m.Commit}
if share && BoundSeat(bs) {
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
digest := ShareDigest(m.Fingerprint, step, status, body.Step.Fetch)
run := TestRunKey(candidate, step, digest)
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
if j, asked := askOfRun[run]; asked {
@@ -168,17 +168,19 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
if status == "" {
continue
}
s := seat.StepResult{Step: step, Status: status, Outcome: seat.StepResultOutcomeWaiting}
s := seat.StepResult{Step: step, Module: m.Module, Status: status, Outcome: seat.StepResultOutcomeWaiting}
if status == seat.StepResultStatusBypass {
s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy"
}
// Composition is the assigner's, an accepted kind step of the seat mesh-wants (§22): it is asked with the others,
// running at once, and its end is heard from that seat as the build seats' are. Review and approval wait.
// running at once, and its end is heard from that seat as the build seats' are; the ask names the runner's seat as its
// asker (H-A-88). Review and approval wait.
if step == seat.StepResultStepComposition && status != seat.StepResultStatusBypass {
s.Seat, s.Outcome, s.Started, s.Fingerprint = wants.Seat, seat.StepResultOutcomeRunning, &now, p.Fingerprint
s.Ask = AskID(startOf(p, now), p.Pipeline, "", step)
asks = append(asks, Asked{Seat: wants.Seat, ID: s.Ask, Ask: wants.StepRun{Branch: p.Branch, Commit: p.Commit, Fingerprint: p.Fingerprint,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: wants.StepRunStepComposition}})
s.Seat, s.Outcome, s.Started, s.Fingerprint = wants.Seat, seat.StepResultOutcomeRunning, &now, m.Fingerprint
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, "", step)
asks = append(asks, Asked{Seat: wants.Seat, ID: s.Ask, Commit: m.Commit, Ask: wants.StepRun{Branch: p.Branch, Commit: m.Commit, Fingerprint: m.Fingerprint,
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Run: p.Run, Step: wants.StepRunStepComposition,
Asker: seat.Seat}})
}
steps = append(steps, s)
}
@@ -200,27 +202,29 @@ type Asked struct {
// never the ask's language, which names the seat; empty for an artifact with none (an image).
Toolchain string
Readers []seat.TestRunReader
// Commit is the mover's commit the ask runs at: a recovery's commit is asked on the build seats' queue delivery (§86).
Commit string
}
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
// artifact's named paths in it when the entry has them.
fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path}
fetch := nodebuildgo.Fetch{Repository: m.Repository, Commit: m.Commit, Dir: e.Path}
if paths := artifactPaths(e, a.Ref); len(paths) > 0 {
fetch.Paths = paths
}
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now,
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint,
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error {
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error {
var first error
for _, a := range asks {
accept, ok := acceptStep[a.Seat]
if to, named := acceptRecovery[a.Seat]; recovery && named {
if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) {
accept = to
}
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
@@ -59,11 +59,11 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
asks := &stepAsks{}
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); !ok || err != nil {
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.4.app"], &p); err != nil {
if err := json.Unmarshal(store["main.4"], &p); err != nil {
t.Fatal(err)
}
var got []string
@@ -94,12 +94,12 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
// module's own entry only.
b, ok := asks.asks[0].(builds.StepRun)
if !ok || asks.at[0] != builds.AcceptSubjectStep || p.Steps[0].Status != seat.StepResultStatusRequired || asks.ids[0] != p.Steps[0].Ask ||
len(b.Entries) != 1 || b.Entries[0] != (builds.BuildEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}) || b.Pipeline != "main.4.app" {
len(b.Entries) != 1 || b.Entries[0] != (builds.BuildEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}) || b.Pipeline != "main.4" {
t.Errorf("the build ask = %+v at %s", asks.asks[0], asks.at[0])
}
a := asks.asks[1].(nodebuildgo.StepAsk)
if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID ||
a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4.app" || a.Step.Artifact != "app:program" ||
a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" ||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 ||
len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" {
t.Errorf("the ask = %+v at %s", a, asks.at[1])
@@ -111,7 +111,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
}
}
// A second start of the revision asks nothing.
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Entries: entries}); ok || err != nil || len(asks.asks) != 7 {
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1"}}}); ok || err != nil || len(asks.asks) != 7 {
t.Errorf("second start = %v, %v, %d asks", ok, err, len(asks.asks))
}
}
@@ -124,12 +124,12 @@ func TestTheBuildHoldsTheApprovalWhateverThePolicyNames(t *testing.T) {
store := created{}
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: &stepAsks{},
Registration: func(string) (catalog.Registration, bool) { return reg, true }}
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); err != nil {
if _, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); err != nil {
t.Fatal(err)
}
var p seat.Pipeline
_ = json.Unmarshal(store["main.4.app"], &p)
if p.Steps[0].Step != seat.StepResultStepBuild || p.Steps[0].Status != seat.StepResultStatusRequired || approvalDue(p) != -1 {
_ = json.Unmarshal(store["main.4"], &p)
if p.Steps[0].Step != seat.StepResultStepBuild || p.Steps[0].Status != seat.StepResultStatusRequired || len(approvalDue(p)) != 0 {
t.Errorf("steps = %+v; want the build first, required, and no approval due", p.Steps)
}
}
@@ -140,10 +140,10 @@ func TestTheBuildStepsEndIsTakenOnItsOneEntry(t *testing.T) {
bs := stepSeatOf(t, builds.Seat)
h, store, _, _, _, _ := fixture()
e := store[pkey]
e.p.Steps = append([]seat.StepResult{{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning,
e.p.Steps = append([]seat.StepResult{{Module: "app", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning,
Ask: "b1", Seat: builds.Seat}}, e.p.Steps...)
store[pkey] = e
raw := `{"id":"b1","pipeline":"main.5.app","step":"build","node":"n1","outcome":"passed","entries":[{"module":"app","commit":"c1","fingerprint":"src2:aa"}]}`
raw := `{"id":"b1","pipeline":"main.5","step":"build","node":"n1","outcome":"passed","entries":[{"module":"app","commit":"c1","fingerprint":"src2:aa"}]}`
ev, err := bs.Decode([]byte(raw))
if err != nil {
t.Fatal(err)
@@ -157,13 +157,13 @@ func TestTheBuildStepsEndIsTakenOnItsOneEntry(t *testing.T) {
}
}
// Composition is asked of the seat mesh-wants with the other steps, running at once, and its end is heard from that seat
// (design 47 G §22, §24): a pipeline whose policy names it places it running with its own ask, a bypassed one asks no one.
// Composition is asked of the seat mesh-wants with the other steps, running at once, naming the runner as its asker, and its
// end is heard from that seat (design 47 G §22, §24; H H-A-88): a pipeline whose policy names it places it running with its own ask, a bypassed one asks no one.
func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) {
reg, entries := placeFixture()
reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Composition: "required", Approval: "required"}}
p := seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Commit: "c1", Fingerprint: "src2:aa", Revision: 7, Source: 4}
steps, asked := Place(p, reg, entries, time.Unix(1, 0), "the-runner")
p := seat.Pipeline{Pipeline: "main.4", Branch: "main", Revision: 7, Source: 4, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
steps, asked := Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner")
var comp seat.StepResult
for _, s := range steps {
if s.Step == seat.StepResultStepComposition {
@@ -181,12 +181,16 @@ func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) {
}
r, ok := got.Ask.(wants.StepRun)
if got == nil || !ok || got.ID != comp.Ask || acceptStep[wants.Seat] != wants.AcceptSubjectStep || r.Step != "composition" ||
r.Commit != "c1" || r.Fingerprint != "src2:aa" || r.Pipeline != "main.4.app" || r.Revision != 7 || r.Source != 4 || r.Module != "app" {
r.Commit != "c1" || r.Fingerprint != "src2:aa" || r.Pipeline != "main.4" || r.Revision != 7 || r.Source != 4 || r.Module != "app" {
t.Fatalf("the composition ask = %+v", got)
}
// H-A-88: the step run names the runner's seat as its asker.
if r.Asker != seat.Seat {
t.Errorf("the composition ask names the asker %q, want %s", r.Asker, seat.Seat)
}
// Its end, from the seat's own subject, is taken like the others, a node's warning becoming the step's why.
bs := stepSeatOf(t, wants.Seat)
ev, err := bs.Decode([]byte(`{"id":"` + comp.Ask + `","pipeline":"main.5.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed","warnings":["n2 asleep"]}`))
ev, err := bs.Decode([]byte(`{"id":"` + comp.Ask + `","pipeline":"main.5","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed","warnings":["n2 asleep"]}`))
if err != nil || ev.Said != "n2 asleep" || ev.Outcome != "passed" || ev.Module != "app" {
t.Fatalf("decoded end = %+v, %v", ev, err)
}
@@ -199,9 +203,8 @@ func TestResumeAsksTheStepsPlacedAndNotEnded(t *testing.T) {
reg, entries := placeFixture()
started := time.Unix(1, 0) // A-297: a step's ask id opens with its pipeline's start, so resuming later places it under the same id
placed := func(branch string, source int64) seat.Pipeline {
p := seat.Pipeline{Pipeline: Key(branch, source, "app"), Branch: branch, Source: source, Module: "app", Commit: "c1", Fingerprint: "src2:aa",
Repository: "o/app", Outcome: seat.PipelineOutcomeRunning, Started: &started}
p.Steps, _ = Place(p, reg, entries, started, "the-runner")
p := seat.Pipeline{Pipeline: Key(branch, source), Branch: branch, Source: source, Outcome: seat.PipelineOutcomeRunning, Started: &started, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}
p.Steps, _ = Place(p, p.Movers[0], reg, entries, started, "the-runner")
return p
}
main4, main5, beta6 := placed("main", 4), placed("main", 5), placed("beta", 6)
@@ -259,7 +262,7 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
Toolchain: func(l string) (*nodebuildgo.ToolchainUsed, error) {
return &nodebuildgo.ToolchainUsed{Language: l, Version: "1.26"}, nil
}}
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil {
if _, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); err != nil {
t.Fatal(err)
}
var program, tools, lib *nodebuildgo.StepRun
@@ -308,7 +311,7 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
"mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}},
"mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}}},
}
steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner")
steps, asks := Place(seat.Pipeline{Pipeline: "main.4", Movers: []seat.Mover{{Module: "app"}}}, seat.Mover{Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner")
want := map[string]struct{ outcome, seat, status string }{
"app:server": {seat.StepResultOutcomeRunning, "node-build-image", "required"},
"app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass},
@@ -406,10 +409,10 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
start := func(source, revision int64) seat.Pipeline {
t.Helper()
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
t.Fatalf("start %d = %v, %v", source, ok, err)
}
p, _, _ := store.Get(Key("main", source, "app"))
p, _, _ := store.Get(Key("main", source))
return p
}
testAsks := func(from int) []string {
@@ -492,10 +495,10 @@ func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Holders: func(string) ([]string, bool) { return c.holders, c.read },
Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil {
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := store.Get("main.4.app")
p, _, _ := store.Get("main.4")
lint := stepNamed(p, "app:program", "lint")
asked := false
for _, id := range asks.ids {
@@ -549,3 +552,50 @@ func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
t.Errorf("halves that disagree name %q", got)
}
}
// R5/AC1 (design 47 H §746): a pipeline of two movers places each mover's steps from its own registration and entry, each
// step naming its mover, and each ask id names its mover, so the same step of two movers is two asks.
func TestEachMoverIsPlacedFromItsOwnRegistrationAndEntry(t *testing.T) {
reg, entries := placeFixture()
lib := catalog.Registration{Module: "lib", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required"}}}
entries["lib"] = catalog.CatalogEntry{Module: "lib", Path: "lib", Artifacts: []catalog.Artifact{{Ref: "lib:lib", Language: "go"}}}
store := created{}
asks := &stepAsks{}
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(m string) (catalog.Registration, bool) {
if m == "lib" {
return lib, true
}
return reg, m == "app"
}}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Entries: entries, Movers: []seat.Mover{
{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}, {Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Repository: "o/lib"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.4"], &p); err != nil {
t.Fatal(err)
}
libLint := stepNamed(p, "lib:lib", seat.StepResultStepLint)
libUnit := stepNamed(p, "lib:lib", seat.StepResultStepUnit)
if libLint.Module != "lib" || libLint.Outcome != seat.StepResultOutcomeBypassed || libUnit.Status != seat.StepResultStatusRequired || libUnit.Fingerprint != "src2:bb" {
t.Errorf("lib's steps lint %+v unit %+v, want them from lib's own policy at its fingerprint", libLint, libUnit)
}
builds := map[string]string{}
for _, st := range p.Steps {
if st.Module == "" {
t.Errorf("the step %s %s names no mover", st.Artifact, st.Step)
}
if st.Step == seat.StepResultStepBuild {
builds[st.Module] = st.Ask
}
}
if len(builds) != 2 || builds["app"] == builds["lib"] || builds["app"] != AskID(*p.Started, "main.4", "app", "", "build") {
t.Errorf("the build asks %v, want one per mover, each id naming its mover", builds)
}
for _, a := range asks.asks {
if b, ok := a.(nodebuildgo.StepAsk); ok && b.Step.Module == "lib" && (b.Step.Commit != "c2" || b.Step.Fetch.Repository != "o/lib" || b.Step.Fetch.Dir != "lib") {
t.Errorf("lib's ask %+v, want lib's commit, repository and directory", b.Step)
}
}
}
@@ -70,8 +70,8 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
}
out := []seat.Pipeline{}
for _, p := range all {
if (in.Module == "" || p.Module == in.Module) && (in.Branch == "" || p.Branch == in.Branch) &&
(in.Commit == "" || p.Commit == in.Commit) && (in.Run == "" || p.Run == in.Run) {
if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) &&
(in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) {
out = append(out, p)
}
}
@@ -98,8 +98,9 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step,
m, _ := MoverOf(p, s.Module)
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: m.Module, Branch: p.Branch, Commit: m.Commit,
Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: m.Fingerprint, Step: s.Step,
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)})
}
}
@@ -184,7 +185,7 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
if raw == nil {
return seat.Pipeline{}, false, nil
}
p, err := seat.DecodePipelinesByBranchRevisionModuleValue(raw)
p, err := seat.DecodePipelinesByBranchSourceValue(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
}
@@ -235,3 +236,13 @@ func sharedOfWait(s seat.StepResult) []seat.Shared {
}
return out
}
// movesCommit says whether a mover of p is at commit.
func movesCommit(p seat.Pipeline, commit string) bool {
for _, m := range p.Movers {
if m.Commit == commit {
return true
}
}
return false
}
@@ -37,19 +37,18 @@ func TestReads(t *testing.T) {
t0 := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC)
t1, t2 := t0.Add(time.Minute), t0.Add(2*time.Minute)
pipes, kept := memKept{}, memKept{}
put(t, pipes, "main.7.alpha", seat.Pipeline{Pipeline: "main.7.alpha", Module: "alpha", Branch: "main", Commit: "c7",
put(t, pipes, "main.7.alpha", seat.Pipeline{Pipeline: "main.7.alpha", Branch: "main",
Source: 7, Outcome: seat.PipelineOutcomeRunning, Started: &t0, Steps: []seat.StepResult{
{Step: seat.StepResultStepLint, Artifact: "alpha:program", Status: "required", Outcome: seat.StepResultOutcomePassed,
{Module: "alpha", Step: seat.StepResultStepLint, Artifact: "alpha:program", Status: "required", Outcome: seat.StepResultOutcomePassed,
Started: &t0, Ended: &t1, Output: "main.7.alpha.lint.alpha:program"},
{Step: seat.StepResultStepUnit, Artifact: "alpha:program", Status: "required", Outcome: seat.StepResultOutcomeCarried,
{Module: "alpha", Step: seat.StepResultStepUnit, Artifact: "alpha:program", Status: "required", Outcome: seat.StepResultOutcomeCarried,
CarriedFrom: "main.6.alpha"},
{Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson,
Started: &t1},
}})
put(t, pipes, "beta.3.alpha", seat.Pipeline{Pipeline: "beta.3.alpha", Module: "alpha", Branch: "beta", Commit: "b3",
Outcome: seat.PipelineOutcomePassed, Started: &t2})
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Module: "gamma", Branch: "main", Commit: "c7",
Outcome: seat.PipelineOutcomeRunning, Started: &t1})
{Module: "alpha", Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson,
Started: &t1}}, Movers: []seat.Mover{{Module: "alpha", Commit: "c7"}}})
put(t, pipes, "beta.3.alpha", seat.Pipeline{Pipeline: "beta.3.alpha", Branch: "beta",
Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}})
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main",
Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}})
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
r := Reads{Pipelines: pipes, Kept: kept}
@@ -68,7 +67,7 @@ func TestReads(t *testing.T) {
}{
{"show answers the pipeline whole and its outputs", func() (string, error) {
a, err := r.Show(seat.ShowInput{Pipeline: "main.7.alpha"})
return a.Pipeline.Module + " " + strings.Join(a.Outputs, ","), err
return a.Pipeline.Movers[0].Module + " " + strings.Join(a.Outputs, ","), err
}, "alpha main.7.alpha.lint.alpha:program"},
{"show refuses a pipeline it does not hold", func() (string, error) {
_, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"})
@@ -134,7 +133,7 @@ func TestReads(t *testing.T) {
// call, never counted from the keys; a fill that cannot be read is an error, not an answer without it (design 47 G §366, §623, A-214).
func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
pipes, kept := memKept{}, memKept{}
put(t, pipes, "main.7.alpha", seat.Pipeline{Pipeline: "main.7.alpha", Module: "alpha"})
put(t, pipes, "main.7.alpha", seat.Pipeline{Pipeline: "main.7.alpha", Movers: []seat.Mover{{Module: "alpha"}}})
fills := []seat.Fill{{Store: seat.StatePipelines, Bytes: 10, Cap: 100}, {Store: seat.StateStepOutputs, Bytes: 20, Cap: 200},
{Store: seat.StateKept, Bytes: 30, Cap: 300}, {Store: seat.StateStepLog, Bytes: 40, Cap: 400}}
r := Reads{Pipelines: pipes, Kept: kept, Fill: func() ([]seat.Fill, error) { return fills, nil }}
@@ -3,7 +3,6 @@ package pipeline
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"maps"
@@ -20,10 +19,13 @@ import (
// RunGroupPrefix is a run on request's candidate group before its id, `run-<id>` (design 47 G A-160).
const RunGroupPrefix = "run-"
// RunKey is the key of a run on request's pipeline, `<branch>.<id>.<module>`: the run's id under the member's named branch
// (design 47 G §420, A-193).
func RunKey(branch, run, module string) string {
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), run, module)
// runToken is the key part that names a run on request's pipeline, `<branch>.run.<id>` (design 47 H H-A-83).
const runToken = "run"
// RunKey is the key of a run on request's one pipeline, `<branch>.run.<id>`, the branch escaped as a key's name is (design
// 47 H H-A-83).
func RunKey(branch, run string) string {
return seat.PipelinesByBranchRunRunKey(keys.EscapeName(branch), run)
}
// NewRunID is a run's id, a random 12-hex id, so two runs of one commit are two pipelines (A-180).
@@ -35,38 +37,45 @@ func NewRunID() (string, error) {
return hex.EncodeToString(b), nil
}
// Run starts a run on request (design 47 G §64, §420; A-180, A-193): it writes each member's pipeline at once, running and
// with no steps, its `run` the run's new id, and says run-started, from which the indexer builds the run's candidate; the
// candidate's revision places the steps (PlaceRun). A member naming no module, commit or branch is refused, the run then
// writing nothing. caller is the one the bus proved. newID makes the run's id (NewRunID).
func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, error)) (seat.RunAnswer, error) {
// Run starts a run on request (design 47 G §64, §420; A-180; H H-A-83, H-A-85): it writes the run's one pipeline at once,
// running and with no steps, its movers the members, its `run` the run's new id and its source the newest source revision
// of its branch as view holds it, and says run-started, from which the indexer builds the run's candidate; the candidate's
// revision places the steps (PlaceRun). A member naming no module, commit or branch, or members naming two branches, is
// refused, the run then writing nothing. caller is the one the bus proved. newID makes the run's id (NewRunID).
func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, error), view func(branch string) (Half, bool)) (seat.RunAnswer, error) {
if len(in.Members) == 0 {
return seat.RunAnswer{}, errors.New("a run names one or more members, each a module at a commit on a branch")
}
branch := in.Members[0].Branch
var movers []seat.Mover
for _, m := range in.Members {
if m.Module == "" || m.Commit == "" || m.Branch == "" {
return seat.RunAnswer{}, fmt.Errorf("a run's member names its module, commit and branch: %+v", m)
}
if m.Branch != branch {
return seat.RunAnswer{}, fmt.Errorf("a run is one pipeline of one branch: its members name %s and %s", branch, m.Branch)
}
if Moves(seat.Pipeline{Movers: movers}, m.Module) {
return seat.RunAnswer{}, fmt.Errorf("a run names %s once", m.Module)
}
movers = append(movers, seat.Mover{Module: m.Module, Commit: m.Commit})
}
var source int64
if view != nil {
if h, ok := view(branch); ok {
source = h.Source
}
}
id, err := newID()
if err != nil {
return seat.RunAnswer{}, fmt.Errorf("the run's id is not made: %w", err)
}
now := s.now()
var written []string
for _, m := range in.Members {
key := RunKey(m.Branch, id, m.Module)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: m.Branch, Module: m.Module, Commit: m.Commit, Run: id,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
raw, err := json.Marshal(p)
if err == nil {
err = s.Pipelines.Create(key, raw)
}
if err != nil {
s.failRun(written, "the run's other pipelines were not written")
return seat.RunAnswer{}, fmt.Errorf("the pipeline %s of the run is not written: %w", key, err)
}
written = append(written, key)
key := RunKey(branch, id)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: branch, Source: source, Movers: movers, Run: id,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
if err := s.create(key, p); err != nil {
return seat.RunAnswer{}, fmt.Errorf("the pipeline %s of the run is not written: %w", key, err)
}
first := in.Members[0]
last := first.Commit
@@ -75,10 +84,10 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err
}
ev := seat.RunStarted{V: 1, At: &now, By: caller, Members: in.Members, Run: id}
if err := s.Say(runnerclient.Subject(seat.EventRunStarted, first.Module, first.Branch, last), ev); err != nil {
s.failRun(written, "run-started was not said, so no candidate is built for it")
s.failRun([]string{key}, "run-started was not said, so no candidate is built for it")
return seat.RunAnswer{}, fmt.Errorf("the run %s is not started: run-started is not said: %w", id, err)
}
return seat.RunAnswer{Run: id, Pipelines: written}, nil
return seat.RunAnswer{Run: id, Pipelines: []string{key}}, nil
}
// failRun ends the run's pipelines written before its start failed, failed and saying why, so none stands running.
@@ -105,11 +114,11 @@ func (s *Starter) failRun(written []string, why string) {
}
}
// PlaceRun places a run's pipelines from its candidate's revision (design 47 G §64, §373, A-180, A-193): ev is the
// catalog-revision of the candidate `run-<id>`, half its entries read whole through the seat's revision verb. Each member's
// pipeline of the run not yet placed takes its fingerprint, the candidate's revision and its steps, by compare-and-set, and
// says pipeline-started; each further entry whose fingerprint the members change (another than in base, the branch's catalog
// half, or absent there) gets a pipeline of the run of its own, placed the same way. Another candidate's revision places
// PlaceRun places a run's pipeline from its candidate's revision (design 47 G §64, §373, A-180; H H-A-83): ev is the
// catalog-revision of the candidate `run-<id>`, half its entries read whole through the seat's revision verb. The run's
// pipeline, not yet placed, takes each member's fingerprint and repository, the candidate's revision, a mover for each
// further entry whose fingerprint the members change (another than in base, the branch's catalog half, or absent there),
// the movers' tiers and their steps, by compare-and-set, and says pipeline-started. Another candidate's revision places
// nothing; a revision heard again places nothing twice. It says how many pipelines it placed.
func (s *Starter) PlaceRun(ev catalog.CatalogRevisionEvent, half Half, base func(branch string) (Half, bool)) (int, error) {
id, ok := strings.CutPrefix(ev.Group, RunGroupPrefix)
@@ -119,28 +128,18 @@ func (s *Starter) PlaceRun(ev catalog.CatalogRevisionEvent, half Half, base func
if s.Store == nil {
return 0, errors.New("the runner holds no pipelines to place a run's in")
}
pkeys, err := s.Store.Keys("*." + id + ".*")
pkeys, err := s.Store.Keys("*." + runToken + "." + id)
if err != nil {
return 0, fmt.Errorf("the pipelines of the run %s are not listed: %w", id, err)
return 0, fmt.Errorf("the pipeline of the run %s is not listed: %w", id, err)
}
slices.Sort(pkeys)
members := map[string]bool{}
branch := ""
for _, key := range pkeys {
if p, _, err := s.Store.Get(key); err == nil && p.Run == id {
members[p.Module] = true
if branch == "" {
branch = p.Branch
}
}
}
if branch == "" {
if len(pkeys) == 0 {
s.logf("the candidate %s's revision %d names a run with no pipeline here: nothing is placed", ev.Group, ev.Revision)
return 0, nil
}
placed := 0
var first error
keep := func(ok bool, err error) {
for _, key := range pkeys {
ok, err := s.placeRunPipeline(key, id, ev, half, base)
if err != nil && first == nil {
first = err
}
@@ -148,42 +147,6 @@ func (s *Starter) PlaceRun(ev catalog.CatalogRevisionEvent, half Half, base func
placed++
}
}
for _, key := range pkeys {
keep(s.placeRunPipeline(key, id, ev.Revision, half))
}
// The dependents (§373): the candidate's entries the members change, beside the members, on the first member's branch.
trunk, ok := base(branch)
if b, err := keys.UnescapeName(ev.Branch); err == nil && b != "" && b != branch {
if h, held := base(b); held {
trunk, ok = h, true
}
}
if !ok {
s.logf("the run %s's dependents are not known: the catalog half of %s is not held, so its members run alone", id, branch)
return placed, first
}
for _, module := range slices.Sorted(maps.Keys(half.Entries)) {
e := half.Entries[module]
if members[module] {
continue
}
if was, held := trunk.Entries[module]; held && was.Fp == e.Fp {
continue
}
key := RunKey(branch, id, module)
now := s.now()
p := seat.Pipeline{V: 1, Pipeline: key, Branch: branch, Module: module, Commit: e.Commit, Run: id,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
raw, err := json.Marshal(p)
if err == nil {
err = s.Pipelines.Create(key, raw)
}
if err != nil && !errors.Is(err, ErrExists) {
keep(false, fmt.Errorf("the dependent %s of the run %s is not written: %w", module, id, err))
continue
}
keep(s.placeRunPipeline(key, id, ev.Revision, half))
}
return placed, first
}
@@ -196,7 +159,7 @@ func (s *Starter) UnplacedRuns() ([]UnplacedRun, error) {
if s.Store == nil {
return nil, errors.New("the runner holds no pipelines to read a run's from")
}
pkeys, err := s.Store.Keys("*.*.*")
pkeys, err := s.Store.Keys("*." + runToken + ".*")
if err != nil {
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
}
@@ -212,8 +175,8 @@ func (s *Starter) UnplacedRuns() ([]UnplacedRun, error) {
return out, nil
}
// placeRunPipeline places one pipeline of the run, written and not yet placed (no revision), from the candidate's half.
func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (bool, error) {
// placeRunPipeline places the run's pipeline, written and not yet placed (no revision), from the candidate's half.
func (s *Starter) placeRunPipeline(key, id string, ev catalog.CatalogRevisionEvent, half Half, base func(branch string) (Half, bool)) (bool, error) {
for {
p, rev, err := s.Store.Get(key)
if err != nil {
@@ -223,19 +186,44 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
return false, nil
}
now := s.now()
e, ok := half.Entries[p.Module]
if !ok {
s.logf("the candidate of the run %s holds no entry of %s: its pipeline is placed with no fingerprint", id, p.Module)
movers := make([]seat.Mover, 0, len(p.Movers))
for _, m := range p.Movers {
e, ok := half.Entries[m.Module]
if !ok {
s.logf("the candidate of the run %s holds no entry of %s: its mover is placed with no fingerprint", id, m.Module)
}
m.Fingerprint, m.Repository = e.Fp, e.Repository
movers = append(movers, m)
}
p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision
// The dependents (§373): the candidate's entries the members change, beside the members.
trunk, held := base(p.Branch)
if b, err := keys.UnescapeName(ev.Branch); err == nil && b != "" && b != p.Branch {
if h, ok := base(b); ok {
trunk, held = h, true
}
}
if !held {
s.logf("the run %s's dependents are not known: the catalog half of %s is not held, so its members run alone", id, p.Branch)
} else {
for _, module := range slices.Sorted(maps.Keys(half.Entries)) {
e := half.Entries[module]
if Moves(seat.Pipeline{Movers: movers}, module) {
continue
}
if was, ok := trunk.Entries[module]; ok && was.Fp == e.Fp {
continue
}
movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
}
p.Movers, p.Revision = Tiers(movers, half.Entries, s.logf), ev.Revision
var asks []Asked
var ended []int
if s.Registration != nil {
if reg, ok := s.Registration(p.Module); ok {
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline is placed with no step", key)
var unregistered []string
asks, unregistered, ended = s.placeMovers(&p, s.Registration, half.Entries, now)
for _, m := range unregistered {
s.logf("%s: %s has no registration yet: its mover is placed with no step", key, m)
}
}
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
@@ -243,16 +231,12 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
} else if err != nil {
return false, fmt.Errorf("the pipeline %s cannot be written: %w", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Revision: p.Revision, Run: id}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
s.sayStartedPipeline(p, now)
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
if err := s.askSteps(key, p.Commit, asks); err != nil {
if err := s.askSteps(key, asks); err != nil {
return true, err
}
}
@@ -9,9 +9,10 @@ import (
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// RN12 (design 47 G §64, §373, §420; A-180, A-193): `run` writes each member's pipeline at once under the run's id and its
// named branch and says run-started; the candidate run-<id>'s revision places each member's steps and a pipeline of each
// entry the members change, says pipeline-started once each, and places nothing twice; a member naming no branch is refused.
// RN12 (design 47 G §64, §373, §420; A-180, A-193; H H-A-83, H-A-85): `run` writes the run's one pipeline at once, keyed
// <branch>.run.<id>, its movers the members, its source its branch's newest, and says run-started; the candidate
// run-<id>'s revision places each member's steps and a mover of each entry the members change, says pipeline-started once,
// and places nothing twice; a member naming no branch, or members of two branches, are refused.
func TestRunOnRequest(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
@@ -28,22 +29,28 @@ func TestRunOnRequest(t *testing.T) {
return nil
}}
id := func() (string, error) { return "0a1b2c3d4e5f", nil }
if _, err := s.Run("sessions", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c1"}}}, id); err == nil {
view := func(b string) (Half, bool) { return Half{Source: 41}, b == "fix/1" }
if _, err := s.Run("sessions", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c1"}}}, id, view); err == nil {
t.Error("a member naming no branch was taken")
}
if _, err := s.Run("sessions", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c1", Branch: "fix/1"}, {Module: "lib", Commit: "c0", Branch: "main"}}}, id, view); err == nil {
t.Error("a run naming two branches was taken: it is one pipeline of one branch")
}
if len(store.memStore) != 0 || len(said) != 0 {
t.Fatalf("a refused run wrote %v and said %v", store.memStore, said)
}
a, err := s.Run("sessions", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c1", Branch: "fix/1"}}}, id)
a, err := s.Run("sessions", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c1", Branch: "fix/1"}}}, id, view)
if err != nil || a.Run != "0a1b2c3d4e5f" || len(a.Pipelines) != 1 {
t.Fatalf("run = %+v, %v", a, err)
}
key := a.Pipelines[0]
if key != RunKey("fix/1", "0a1b2c3d4e5f", "app") {
t.Errorf("the member's pipeline is %s, want the run's id under its named branch", key)
// H-A-83, H-A-85: a run is one pipeline keyed <branch>.run.<id>, its movers the members, its source the branch's newest.
if key != "fix%2F1.run.0a1b2c3d4e5f" && key != RunKey("fix/1", "0a1b2c3d4e5f") {
t.Errorf("the run's pipeline is %s, want <branch>.run.<id>", key)
}
if p, _, _ := store.Get(key); p.Run != a.Run || p.Outcome != seat.PipelineOutcomeRunning || len(p.Steps) != 0 || p.Branch != "fix/1" || p.Source != 0 {
t.Errorf("the member's pipeline %s = %+v", key, p)
if p, _, _ := store.Get(key); p.Run != a.Run || p.Outcome != seat.PipelineOutcomeRunning || len(p.Steps) != 0 || p.Branch != "fix/1" ||
p.Source != 41 || len(p.Movers) != 1 || p.Movers[0].Module != "app" {
t.Errorf("the run's pipeline %s = %+v", key, p)
}
if len(said) != 1 || !strings.HasPrefix(said[0], seat.EventRunStarted+".app.") || !strings.HasSuffix(said[0], ".c1") {
t.Errorf("said %v, want run-started on the member's module, branch and commit", said)
@@ -65,19 +72,14 @@ func TestRunOnRequest(t *testing.T) {
}
said = nil
n, err := s.PlaceRun(ev, Half{Revision: 12, Entries: cand}, base)
if err != nil || n != 2 {
t.Fatalf("the candidate placed %d (%v), want the member and the dependent lib", n, err)
if err != nil || n != 1 {
t.Fatalf("the candidate placed %d (%v), want the run's one pipeline", n, err)
}
p, _, _ := store.Get(key)
if p.Fingerprint != "src2:new" || p.Revision != 12 || len(p.Steps) == 0 {
t.Errorf("the member's pipeline placed = %+v", p)
}
dep, _, err := store.Get(RunKey("fix/1", a.Run, "lib"))
if err != nil || dep.Run != a.Run || dep.Fingerprint != "src2:lib2" || len(dep.Steps) == 0 {
t.Errorf("the dependent's pipeline = %+v (%v)", dep, err)
}
if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil {
t.Error("an entry the members do not change got a pipeline")
app, _ := MoverOf(p, "app")
lib, _ := MoverOf(p, "lib")
if app.Fingerprint != "src2:new" || lib.Fingerprint != "src2:lib2" || Moves(p, "same") || p.Revision != 12 || !placed(p, "app") || !placed(p, "lib") {
t.Errorf("the run's pipeline placed = %+v, want the member and the dependent lib as movers, not same", p)
}
// Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a).
started, steps := 0, 0
@@ -85,7 +87,7 @@ func TestRunOnRequest(t *testing.T) {
started += strings.Count(x, seat.EventPipelineStarted+".")
steps += strings.Count(x, seat.EventStepStarted+".")
}
if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
if started != 1 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
t.Errorf("placing said %v and asked %d steps", said, len(asks.ids))
}
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
@@ -103,16 +105,16 @@ func TestRunOnRequest(t *testing.T) {
// pipelines written, so none stands running.
said = nil
two, err := s.Run("operator", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c2", Branch: "main"}, {Module: "lib", Commit: "c3", Branch: "main"}}},
func() (string, error) { return "ffffffffffff", nil })
if err != nil || len(two.Pipelines) != 2 || len(said) != 1 || said[0] != seat.EventRunStarted+".app.main.run-ffffffffffff" {
func() (string, error) { return "ffffffffffff", nil }, view)
if err != nil || len(two.Pipelines) != 1 || len(said) != 1 || said[0] != seat.EventRunStarted+".app.main.run-ffffffffffff" {
t.Errorf("a run of two: %+v (%v), said %v", two, err, said)
}
s.Say = func(string, any) error { return errors.New("bus away") }
three, err := s.Run("operator", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c4", Branch: "main"}}}, func() (string, error) { return "eeeeeeeeeeee", nil })
three, err := s.Run("operator", seat.RunInput{Members: []seat.Member{{Module: "app", Commit: "c4", Branch: "main"}}}, func() (string, error) { return "eeeeeeeeeeee", nil }, view)
if err == nil || three.Run != "" {
t.Errorf("a run whose run-started was not said answered %+v", three)
}
if p, _, _ := store.Get(RunKey("main", "eeeeeeeeeeee", "app")); p.Outcome != seat.PipelineOutcomeFailed {
if p, _, _ := store.Get(RunKey("main", "eeeeeeeeeeee")); p.Outcome != seat.PipelineOutcomeFailed {
t.Errorf("its pipeline is %s, want failed", p.Outcome)
}
}
@@ -78,7 +78,7 @@ func (h *Hearer) runOfAsk(ask string) (string, bool) {
if h.TestRuns == nil || ask == "" {
return "", false
}
pkeys, err := h.Store.Keys("*.*.*")
pkeys, err := allKeys(h.Store)
if err != nil {
return "", false
}
@@ -27,29 +27,102 @@ type Created interface {
Create(key string, value []byte) error
}
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
// what the catalog half of that branch's view says of the module.
// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's
// call and catalog graph revision, its movers, and the catalog half of that branch.
type Begin struct {
Branch string
Source int64
Module string
Commit string
Fp string
Group string
Repository string
Revision int64
Branch string
Source int64
Call string
Revision int64
// Movers are the take-in's entries, each with its tier (§746).
Movers []seat.Mover
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
Entries map[string]catalog.CatalogEntry
// Published is the package's publish that started the pipeline, nil for any other start (A-202).
// Published is the package's publish that started the pipeline, nil for any other start (§747).
Published *seat.Published
// Key is the pipeline's key when it is not <branch>.<source> (a publish's, §747); empty for a take-in's.
Key string
// unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336).
unheld bool
}
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
// name is (§27).
func Key(branch string, source int64, module string) string {
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
// Key is a take-in's pipeline key in the state pipelines, `<branch>.<source>`, the branch escaped as a key's name is
// (design 47 H §746).
func Key(branch string, source int64) string {
return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))
}
// PublishedKey is the key of a pipeline a package's publish starts, `<branch>.<source>.published.<package>.<version>`,
// each part escaped as a key's name is (design 47 H §747).
func PublishedKey(branch string, source int64, pkg, version string) string {
return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10),
keys.EscapeName(pkg), keys.EscapeName(version))
}
// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's.
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"}
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
func AllKeys(store interface {
Keys(string) ([]string, error)
}) ([]string, error) {
return allKeys(store)
}
func allKeys(store interface {
Keys(string) ([]string, error)
}) ([]string, error) {
var out []string
for _, f := range keyForms {
k, err := store.Keys(f)
if err != nil {
return nil, err
}
out = append(out, k...)
}
return out, nil
}
// branchKeys are the keys of every pipeline of one watched branch, of each key form.
func branchKeys(store interface {
Keys(string) ([]string, error)
}, branch string) ([]string, error) {
var out []string
for _, f := range keyForms {
_, rest, _ := strings.Cut(f, ".")
k, err := store.Keys(keys.EscapeName(branch) + "." + rest)
if err != nil {
return nil, err
}
out = append(out, k...)
}
return out, nil
}
// MoverOf is the mover of p whose module is module.
func MoverOf(p seat.Pipeline, module string) (seat.Mover, bool) {
for _, m := range p.Movers {
if m.Module == module {
return m, true
}
}
return seat.Mover{}, false
}
// Moves says whether p moves module.
func Moves(p seat.Pipeline, module string) bool {
_, ok := MoverOf(p, module)
return ok
}
// placed says whether mover m of p has its steps placed: every placed mover has its build step at least.
func placed(p seat.Pipeline, module string) bool {
for _, s := range p.Steps {
if s.Module == module {
return true
}
}
return false
}
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
@@ -87,8 +160,6 @@ type Starter struct {
Recovered func(commit string) (bool, error)
// Store is the state pipelines read and written by compare-and-set, where a run's pipelines are placed (PlaceRun).
Store Updater
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
Supersede func(newer seat.Pipeline)
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
// (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the
// settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends
@@ -158,13 +229,13 @@ func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
// the kept results and toolchain are carried and each ask is put.
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
func (s *Starter) askSteps(key string, asks []Asked) error {
placed := asks
asks = s.share(key, asks)
var err error
if len(asks) > 0 {
s.carry(key, asks)
err = askAll(s.Steps, key, asks, s.recovered(key, commit))
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) })
}
s.sayStarted(key, placed)
return err
@@ -202,9 +273,9 @@ func (s *Starter) sayStarted(key string, asks []Asked) {
}
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp,
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
Steps: []seat.StepResult{}, Published: b.Published}
movers := append([]seat.Mover(nil), b.Movers...)
return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published}
}
func (s *Starter) create(key string, p seat.Pipeline) error {
@@ -215,17 +286,56 @@ func (s *Starter) create(key string, p seat.Pipeline) error {
return s.Pipelines.Create(key, raw)
}
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers.
func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
return seat.PipelineStarted{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call,
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
}
// sayStartedPipeline says p's pipeline-started on <branch>.<source> (§746).
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
if s.Say == nil {
return
}
if err := s.Say(runnerclient.SourceSubject(seat.EventPipelineStarted, p.Branch, p.Source), started(p, at)); err != nil {
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
}
}
// placeMovers places the steps of each mover of p that has none and whose registration reg names (§746): each mover's from
// its own registration and entry. It answers the asks to put, the movers still without a registration, and the steps the
// placing ended at once (holderless).
func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog.Registration, bool), entries map[string]catalog.CatalogEntry, now time.Time) ([]Asked, []string, []int) {
var asks []Asked
var unregistered []string
for _, m := range p.Movers {
if placed(*p, m.Module) {
continue
}
r, ok := reg(m.Module)
if !ok {
unregistered = append(unregistered, m.Module)
continue
}
steps, a := Place(*p, m, r, entries, now, s.Name)
p.Steps = append(p.Steps, steps...)
asks = append(asks, a...)
}
asks, ended := s.holderless(p, asks, now)
return asks, unregistered, ended
}
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
// start writes nothing, §54).
// start writes nothing, §746).
func (s *Starter) Start(b Begin) (bool, error) {
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
if b.Branch == "" || len(b.Movers) == 0 || b.Source <= 0 {
return false, fmt.Errorf("a pipeline names its watched branch, source revision and one or more movers: %+v", b)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
now := s.now()
key := b.Key
if key == "" {
key = Key(b.Branch, b.Source)
}
key := Key(b.Branch, b.Source, b.Module)
if err := s.Fills.refuseStart(key); err != nil {
return false, err
}
@@ -235,11 +345,10 @@ func (s *Starter) Start(b Begin) (bool, error) {
if b.unheld {
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
} else if s.Registration != nil {
if reg, ok := s.Registration(b.Module); ok {
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
var unregistered []string
asks, unregistered, ended = s.placeMovers(&p, s.Registration, b.Entries, now)
for _, m := range unregistered {
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
}
}
if err := s.create(key, p); errors.Is(err, ErrExists) {
@@ -248,23 +357,12 @@ func (s *Starter) Start(b Begin) (bool, error) {
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published}
commitOrGroup := b.Commit
if b.Group != "" {
commitOrGroup = b.Group
}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if s.Supersede != nil {
s.Supersede(p)
}
s.sayStartedPipeline(p, now)
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
if err := s.askSteps(key, b.Commit, asks); err != nil {
if err := s.askSteps(key, asks); err != nil {
return true, err
}
}
@@ -280,14 +378,11 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if s.Steps == nil || s.Registration == nil {
return 0, nil
}
pkeys, err := store.Keys("*.*.*")
pkeys, err := allKeys(store)
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
now := s.now()
asked := 0
var first error
for _, key := range pkeys {
@@ -302,10 +397,6 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" {
continue
}
reg, ok := s.Registration(p.Module)
if !ok {
continue
}
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
running, own := map[string]bool{}, map[string]bool{}
@@ -314,24 +405,30 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
}
}
_, shared := Place(p, reg, half.Entries, now, s.Name)
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
var due []Asked
for _, a := range shared {
if running[a.ID] && !own[a.ID] {
due = append(due, a)
for _, m := range p.Movers {
reg, ok := s.Registration(m.Module)
if !ok {
continue
}
}
for _, a := range unshared {
if running[a.ID] && own[a.ID] {
a.Run = ""
due = append(due, a)
_, shared := Place(p, m, reg, half.Entries, now, s.Name)
_, unshared := place(p, m, reg, half.Entries, now, s.Name, false)
for _, a := range shared {
if running[a.ID] && !own[a.ID] {
due = append(due, a)
}
}
for _, a := range unshared {
if running[a.ID] && own[a.ID] {
a.Run = ""
due = append(due, a)
}
}
}
if len(due) == 0 {
continue
}
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
if err := s.askSteps(key, due); err != nil && first == nil {
first = err
}
asked += len(due)
@@ -339,48 +436,50 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
return asked, first
}
// Registered places, on a registration-changed of reg's module, the steps of every running pipeline of that module that has
// none, from the registration (design 47 G §21, §23, A-327): a pipeline started before its module had a registration. Each is
// written by compare-and-set, so it is placed once, its steps asked, and said; one held for the runner's new version is left to
// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, said, until that
// half is held (Waiting). It says how many it placed.
// Registered places, on a registration-changed of reg's module, the steps of that module's mover in every running pipeline
// that moves it with none placed, from the registration (design 47 G §21, §23, A-327): a pipeline started before its module
// had a registration. Each is written by compare-and-set, so it is placed once, its steps asked, and said; a run on request
// is left to its candidate, and one whose branch has no catalog half yet keeps waiting, said, until that half is held
// (Waiting). It says how many pipelines it placed in.
func (s *Starter) Registered(store Updater, reg catalog.Registration, view func(branch string) (Half, bool)) (int, error) {
if reg.Module == "" {
return 0, nil
}
return s.placeWaiting(store, "*.*."+reg.Module, func(p seat.Pipeline) (catalog.Registration, bool) {
return reg, p.Module == reg.Module
}, view, "the registration of "+reg.Module+", which arrived after the pipeline started")
return s.placeWaiting(store, func(p seat.Pipeline) bool { return Moves(p, reg.Module) && !placed(p, reg.Module) },
func(module string) (catalog.Registration, bool) { return reg, module == reg.Module },
view, "the registration of "+reg.Module+", which arrived after the pipeline started")
}
// Waiting places, from the registrations held now, the steps of every running pipeline of branch that has none (A-327): a
// registration heard before the runner could place from it, or before the branch's catalog half was held, is not lost, since
// the half becoming held and the registrations' first view being held each place what waited on them. An empty branch is
// every branch. It says how many it placed.
// Waiting places, from the registrations held now, the steps of every mover with none of every running pipeline of branch
// (A-327): a registration heard before the runner could place from it, or before the branch's catalog half was held, is not
// lost, since the half becoming held and the registrations' first view being held each place what waited on them. An empty
// branch is every branch. It says how many pipelines it placed in.
func (s *Starter) Waiting(store Updater, branch string, view func(branch string) (Half, bool)) (int, error) {
if s.Registration == nil {
return 0, nil
}
return s.placeWaiting(store, "*.*.*", func(p seat.Pipeline) (catalog.Registration, bool) {
return s.placeWaiting(store, func(p seat.Pipeline) bool {
if branch != "" && p.Branch != branch {
return catalog.Registration{}, false
return false
}
return s.Registration(p.Module)
}, view, "the registration held when its branch's half or the registrations became held")
for _, m := range p.Movers {
if !placed(p, m.Module) {
return true
}
}
return false
}, s.Registration, view, "the registration held when its branch's half or the registrations became held")
}
// placeWaiting places the steps of each running pipeline under pattern that has none and is not a run on request,
// and whose registration reg names, from that registration and its branch's catalog half (Registered, Waiting).
func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipeline) (catalog.Registration, bool), view func(branch string) (Half, bool), from string) (int, error) {
pkeys, err := store.Keys(pattern)
// placeWaiting places the steps of each mover with none of each running pipeline due names that is not a run on request,
// from the registration reg names and its branch's catalog half (Registered, Waiting).
func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg func(module string) (catalog.Registration, bool), view func(branch string) (Half, bool), from string) (int, error) {
pkeys, err := allKeys(store)
if err != nil {
return 0, fmt.Errorf("the pipelines %s are not listed: %w", pattern, err)
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
placed := 0
now := s.now()
count := 0
var first error
note := func(err error) {
if first == nil {
@@ -393,11 +492,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
note(fmt.Errorf("the pipeline %s is not read: %w", key, err))
continue
}
if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || len(p.Steps) > 0 {
continue
}
r, ok := reg(p)
if !ok {
if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || !due(p) {
continue
}
half, ok := view(p.Branch)
@@ -409,24 +504,20 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision)
continue
}
var asks []Asked
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
asks, ended := s.holderless(&p, asks, now)
before := len(p.Steps)
asks, _, ended := s.placeMovers(&p, reg, half.Entries, now)
if len(p.Steps) == before {
continue
}
if _, err := store.Update(key, p, rev); err != nil {
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
continue
}
placed++
s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps), from)
// A pipeline whose steps were never asked is started by this placing, one a runner before this one set aside included
// (A-446a); the pipeline-started of its start, said already, is said again harmlessly.
if s.Say != nil {
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Source: p.Source, Module: p.Module,
Commit: p.Commit, Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Published: p.Published}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, commitOrGroup(p)), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
}
count++
s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps)-before, from)
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
// said already, is said again harmlessly.
s.sayStartedPipeline(p, now)
if len(ended) > 0 {
if err := s.endedAtPlacing(p, ended); err != nil {
note(err)
@@ -434,12 +525,12 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
continue
}
if s.Steps != nil && len(asks) > 0 {
if err := s.askSteps(key, p.Commit, asks); err != nil {
if err := s.askSteps(key, asks); err != nil {
note(err)
}
}
}
return placed, first
return count, first
}
// recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since
@@ -503,14 +594,60 @@ func (s *Starter) logf(format string, args ...any) {
}
}
// TakenInSubject is where any module's take-in.taken is said, `mesh.mod.<module>.event.take-in.taken`: the runner names no
// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54).
const TakenInSubject = "mesh.mod.*.event.take-in.taken"
// TakenIn starts, for a take-in.taken with entries, exactly one pipeline keyed `<branch>.<source>` by compare-and-set on its
// absence, whose movers are the event's entries, each with its tier (design 47 H §746). A take-in with no entries, or a
// skip, starts nothing; a second hearing writes nothing. view is the catalog half of the event's branch, read for the
// entries the steps and the tiers are made from.
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
if len(ev.Entries) == 0 {
s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source)
return 0, nil
}
if ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: branch %q source %d", ev.Branch, ev.Source)
}
// A take-in heard before its branch's half is held starts its pipeline with no steps, never placed from empty entries
// (§200, A-336); Waiting places them once the half is held. It starts at the revision the take-in made (A-448): while the
// view holds an older one it starts with no step, which Waiting places once the view reaches it.
half, held := view(ev.Branch)
if held && half.Revision < ev.Revision {
s.logf("the take-in of %s source %d made revision %d and the runner's view holds %d: its pipeline starts with no step, placed once the view reaches it (§200)", ev.Branch, ev.Source, ev.Revision, half.Revision)
held = false
}
movers := make([]seat.Mover, 0, len(ev.Entries))
for _, e := range ev.Entries {
movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository})
}
movers = Tiers(movers, half.Entries, s.logf)
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: ev.Call, Revision: ev.Revision, Movers: movers,
Entries: half.Entries, unheld: !held})
started := 0
if ok {
started = 1
}
// The view was read before the start wrote: when the pipeline was held back for it and it has reached the revision since,
// it is placed now, not left for an advance that already came (A-448, §200).
if !held && ok && ev.Revision > 0 && s.Store != nil {
if again, held := view(ev.Branch); held && again.Revision >= ev.Revision {
if _, werr := s.Waiting(s.Store, ev.Branch, view); werr != nil && err == nil {
err = werr
}
}
}
return started, err
}
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
// a package A imports (§53).
const outdatedKind = 1
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
// dependent the half does not hold is said and started with what the event names.
// Published starts, for a package-published, one pipeline keyed `<branch>.<source>.published.<package>.<version>` whose
// movers are the dependents the catalog's `outdated` names behind for that package on that branch by kind 1 (design 47 G
// §53; H §747). outdated is the kept call of the verb; view the catalog half of the publication's branch.
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
@@ -519,95 +656,24 @@ func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.Ou
if err != nil {
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
}
// A publish heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries
// (§200, A-336); Waiting places them once the half is held.
half, held := view(ev.Branch)
started := 0
var first error
var movers []seat.Mover
for _, o := range a.Behind {
if o.Kind != outdatedKind || o.Module == "" {
if o.Kind != outdatedKind || o.Module == "" || Moves(seat.Pipeline{Movers: movers}, o.Module) {
continue
}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision, Entries: half.Entries,
Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, unheld: !held}
if e, ok := half.Entries[o.Module]; ok {
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
} else if held {
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
}
ok, err := s.Start(b)
if err != nil && first == nil {
first = err
}
if ok {
started++
}
e := half.Entries[o.Module]
movers = append(movers, seat.Mover{Module: o.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
return started, first
}
// TakenInSubject is where any module's take-in.taken is said, `mesh.mod.<module>.event.take-in.taken`: the runner names no
// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54).
const TakenInSubject = "mesh.mod.*.event.take-in.taken"
// TakenIn starts, for a take-in.taken, one pipeline per entry of the source revision, keyed by its watched branch, that
// source revision and the module (design 47 G §44, §54, §476; ADR 0666 §13). A hand drop takes nothing in and starts
// nothing; a second start of an entry, a package-published having started it first, writes nothing (§54). view is the
// catalog half of the event's branch, read only for the revision its pipelines are made from.
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
if ev.Dropped {
s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By)
if len(movers) == 0 {
return 0, nil
}
if len(ev.Entries) == 0 {
// A taken event that took nothing in (a group's end, one said before G and read again) starts nothing (§493).
s.logf("the take-in %s names no entries: it starts no pipeline", ev.Key)
return 0, nil
movers = Tiers(movers, half.Entries, s.logf)
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Revision: half.Revision, Movers: movers, Entries: half.Entries,
Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, unheld: !held,
Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)})
if ok {
return 1, err
}
if ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source)
}
// A take-in heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries
// (§200, A-336); Waiting places them once the half is held.
half, held := view(ev.Branch)
// The pipelines start at the revision the take-in made, never at the one the runner's own view holds (A-448): while the
// view holds an older one they start with no step, which Waiting places once the view reaches it (§200). An event with
// no revision (said before A-448) starts at the view's, as it did, and says so.
revision := ev.Revision
if revision <= 0 {
revision = half.Revision
s.logf("the take-in %s of %s names no catalog graph revision: its pipelines start at the revision the runner's view holds, %d", ev.Key, ev.Branch, revision)
} else if held && half.Revision < revision {
s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision)
held = false
}
started := 0
var first error
for _, e := range ev.Entries {
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
Group: ev.Group, Repository: e.Repository, Revision: revision, Entries: half.Entries, unheld: !held})
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
// The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since,
// they are placed now, not left for an advance that already came (A-448, §200).
if !held && started > 0 && revision > 0 && s.Store != nil {
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
if _, err := s.Waiting(s.Store, ev.Branch, view); err != nil && first == nil {
first = err
}
}
}
return started, first
}
func commitOrGroup(p seat.Pipeline) string {
if p.Group != "" {
return p.Group
}
return p.Commit
return 0, err
}
@@ -8,7 +8,6 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -22,13 +21,12 @@ func (c created) Create(key string, value []byte) error {
return nil
}
// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start
// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names
// behind by kind 1, keyed by the publishing branch and revision.
// The starts (design 47 G §53, §54; H §746, §747): a pipeline's key written by compare-and-set on its absence, so a second
// start of the same key writes nothing; a publish starts one pipeline keyed <branch>.<source>.published.<package>.<version>
// whose movers are the dependents outdated names behind by kind 1; a branch holding a dot is escaped in the key.
func TestStarts(t *testing.T) {
store := created{}
var said []string
var superseding []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev, started := body.(seat.PipelineStarted)
if !started { // only pipeline-started is counted here
@@ -39,7 +37,7 @@ func TestStarts(t *testing.T) {
t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published)
}
return nil
}, Supersede: func(newer seat.Pipeline) { superseding = append(superseding, newer.Pipeline) }}
}}
view := func(branch string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
@@ -51,46 +49,44 @@ func TestStarts(t *testing.T) {
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
}
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
if err != nil || n != 2 {
t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err)
if err != nil || n != 1 {
t.Fatalf("a publish started %d (%v), want one pipeline", n, err)
}
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
t.Errorf("outdated asked %+v, want the package on its branch", asked)
}
key := PublishedKey("main", 41, "mesh-sdk:go", "0.1.16")
var p seat.Pipeline
if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fingerprint != "src2:aa" || p.Revision != 9 ||
p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 || p.Published == nil || p.Published.Version != "0.1.16" {
t.Errorf("the pipeline main.41.app = %+v (%v)", p, err)
if err := json.Unmarshal(store[key], &p); err != nil || len(p.Movers) != 2 || p.Revision != 9 || p.Outcome != seat.PipelineOutcomeRunning ||
p.Source != 41 || p.Published == nil || p.Published.Version != "0.1.16" || Moves(p, "ordered") {
t.Fatalf("the pipeline %s = %+v (%v)", key, p, err)
}
// Each pipeline written supersedes the older ones it replaces (§31).
if len(superseding) != 2 {
t.Errorf("the publish's pipelines superseding: %v, want both", superseding)
if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" {
t.Errorf("the mover app = %+v", app)
}
if _, ok := store["main.41.ordered"]; ok {
t.Error("an ordering edge starts no pipeline")
}
// The take-in of the same source revision and module is the same pipeline: the second start writes nothing.
before := string(store["main.41.app"])
ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"})
if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 {
before := string(store[key])
ok, err := s.Start(Begin{Branch: "main", Source: 41, Key: key, Movers: []seat.Mover{{Module: "app", Commit: "c2"}}})
if ok || err != nil || string(store[key]) != before || len(said) != 1 {
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
}
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") {
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.41 "+key) {
t.Errorf("pipeline-started said on %q", said[0])
}
s.Say = func(string, any) error { return nil }
// A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused.
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil {
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil {
t.Fatal(err)
}
if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" {
t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app"))
if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" {
t.Errorf("the key of release.1 is %q", Key("release.1", 3))
}
if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil {
if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil {
t.Error("a start with no source revision is refused")
}
if _, err := s.Start(Begin{Branch: "main", Source: 6}); err == nil {
t.Error("a start with no mover is refused")
}
s.Pipelines = failing{}
if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil {
if _, err := s.Start(Begin{Branch: "main", Source: 5, Movers: []seat.Mover{{Module: "app"}}}); err == nil {
t.Error("a write the bus refuses is an error")
}
}
@@ -117,39 +113,46 @@ type failing struct{}
func (failing) Create(string, []byte) error { return errors.New("bus away") }
// A take-in.taken starts one pipeline per entry of its source revision (design 47 G §44, §54, §476); a hand drop starts
// none, and a publish that started an entry first leaves the take-in's start of it writing nothing.
func TestTakenInStarts(t *testing.T) {
// R4/AC1 (design 47 H §746, acceptance test 112): a take-in.taken naming mesh-sdk and one importer starts one pipeline
// keyed <branch>.<source> by compare-and-set, its movers the event's entries, at the take-in's revision, and says one
// pipeline-started on <branch>.<source> naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A
// take-in with no entries and one naming no source revision start nothing.
func TestATakeInIsOnePipeline(t *testing.T) {
store := created{}
var said []string
var said []seat.PipelineStarted
var subjects []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
if _, pins := body.(delivery.Pins); !pins {
said = append(said, subject)
if ev, ok := body.(seat.PipelineStarted); ok {
said, subjects = append(said, ev), append(subjects, subject)
}
return nil
}}
view := func(string) (Half, bool) { return Half{Revision: 12}, true }
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{
{Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}}
if _, err := s.Start(Begin{Branch: "main", Source: 7, Module: "b", Commit: "c2"}); err != nil {
t.Fatal(err)
ev := catalog.TakeInTaken{Branch: "main", Source: 7, Call: "0123456789abcdef", Revision: 12, Entries: []catalog.TakeInEntry{
{Module: "mesh-sdk", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/sdk"}, {Module: "app", Commit: "c2", Fingerprint: "src2:bb", Repository: "o/app"}}}
for i := range 2 {
n, err := s.TakenIn(ev, view)
if err != nil || n != 1-i {
t.Fatalf("hearing %d started %d (%v)", i+1, n, err)
}
}
n, err := s.TakenIn(ev, view)
if err != nil || n != 1 || len(said) != 2 {
t.Fatalf("a take-in started %d (%v), said %v: want a only, b being started already", n, err, said)
if len(store) != 1 || len(said) != 1 {
t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said))
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.7.a"], &p); err != nil || p.Commit != "c1" || p.Group != "g1" || p.Revision != 12 || p.Fingerprint != "src2:aa" {
t.Errorf("the pipeline main.7.a = %+v (%v)", p, err)
if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 {
t.Fatalf("the pipeline main.7 = %+v (%v)", p, err)
}
if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Dropped: true, By: "op", Branch: "main", Source: 8, Entries: ev.Entries}, view); n != 0 || err != nil {
t.Errorf("a hand drop started %d (%v)", n, err)
if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 {
t.Errorf("the mover mesh-sdk = %+v", m)
}
if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main", Entries: ev.Entries}, view); err == nil {
if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.7" || said[0].Call != ev.Call || len(said[0].Movers) != 2 {
t.Errorf("pipeline-started %+v on %s", said[0], subjects[0])
}
if _, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Entries: ev.Entries}, view); err == nil {
t.Error("a take-in naming no source revision is refused")
}
// §493: a taken event with no entries (a group's end, one said before G) starts nothing and is not heard again.
if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Key: "m.9"}, view); n != 0 || err != nil {
if n, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Source: 9}, view); n != 0 || err != nil {
t.Errorf("a take-in with no entries started %d (%v)", n, err)
}
}
@@ -165,16 +168,16 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(subject string, _ any) error { said = append(said, subject); return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
taken := catalog.TakeInTaken{Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the take-in started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.4.app"); len(p.Steps) != 0 || len(asks.asks) != 0 {
if p, _, _ := store.Get("main.4"); len(p.Steps) != 0 || len(asks.asks) != 0 {
t.Fatalf("before its registration the pipeline has %d steps and %d asks, want none", len(p.Steps), len(asks.asks))
}
registered = true
n, err := s.Registered(store.memStore, reg, view)
p, _, _ := store.Get("main.4.app")
p, _, _ := store.Get("main.4")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Outcome != seat.PipelineOutcomeRunning {
t.Fatalf("the registration placed %d (%v): %d steps, %d asks, want the pipeline's steps placed and asked", n, err, len(p.Steps), len(asks.asks))
}
@@ -187,7 +190,7 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
// lost: the half becoming held, or the registrations' first view, places the waiting pipeline once (Waiting).
held := false
late := func(string) (Half, bool) { return Half{Entries: entries}, held }
taken = catalog.TakeInTaken{Key: "m.2", Branch: "next", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}}
taken = catalog.TakeInTaken{Branch: "next", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}}
registered = false
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the take-in on next started %d (%v)", n, err)
@@ -202,7 +205,7 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
t.Errorf("the half of main placed %d (%v) of next's pipelines", n, err)
}
n, err = s.Waiting(store.memStore, "next", late)
p, _, _ = store.Get("next.5.app")
p, _, _ = store.Get("next.5")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
t.Fatalf("the half of next becoming held placed %d (%v): %d steps, %d asks more, want its steps placed and asked", n, err, len(p.Steps), len(asks.asks)-before)
}
@@ -214,16 +217,16 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
// A-336 (§200): a take-in heard before its branch's half is held starts its pipelines with no steps, never placed from
// empty entries, the registration held or not; the half becoming held places them (Waiting).
held = false
taken = catalog.TakeInTaken{Key: "m.3", Branch: "next", Source: 6, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:cc"}}}
taken = catalog.TakeInTaken{Branch: "next", Source: 6, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:cc"}}}
if n, err := s.TakenIn(taken, late); n != 1 || err != nil {
t.Fatalf("the take-in before the half started %d (%v)", n, err)
}
if p, _, _ := store.Get("next.6.app"); len(p.Steps) != 0 {
if p, _, _ := store.Get("next.6"); len(p.Steps) != 0 {
t.Fatalf("a take-in before its branch's half placed %d steps from empty entries, want none", len(p.Steps))
}
held = true
n, err = s.Waiting(store.memStore, "next", late)
p, _, _ = store.Get("next.6.app")
p, _, _ = store.Get("next.6")
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps))
}
@@ -235,12 +238,12 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}, outdated, late); n != 1 || err != nil {
t.Fatalf("the publish before the half started %d (%v)", n, err)
}
if p, _, _ := store.Get("next.7.app"); len(p.Steps) != 0 {
if p, _, _ := store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")); len(p.Steps) != 0 {
t.Fatalf("a publish before its branch's half placed %d steps from empty entries, want none", len(p.Steps))
}
held = true
n, err = s.Waiting(store.memStore, "next", late)
p, _, _ = store.Get("next.7.app")
p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17"))
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps))
}
@@ -248,18 +251,18 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
var old seat.Pipeline
if err := json.Unmarshal([]byte(`{"pipeline":"main.940.app","branch":"main","module":"app","commit":"c9","source":940,"outcome":"running",`+
if err := json.Unmarshal([]byte(`{"pipeline":"main.940","branch":"main","movers":[{"module":"app","commit":"c9","fingerprint":"src2:new","repository":"","tier":1}],"source":940,"outcome":"running",`+
`"held":{"since":"2026-11-15T18:05:00Z","fingerprint":"src2:new"}}`), &old); err != nil {
t.Fatal(err)
}
store.memStore["main.940.app"] = struct {
store.memStore["main.940"] = struct {
p seat.Pipeline
rev uint64
}{old, 1}
before, saidBefore := len(asks.asks), len(said)
n, err = s.Waiting(store.memStore, "main", late)
p, _, _ = store.Get("main.940.app")
if len(said) != saidBefore+1 || !strings.HasPrefix(said[saidBefore], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c9") {
p, _, _ = store.Get("main.940")
if len(said) != saidBefore+1 || !strings.HasPrefix(said[saidBefore], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.940") {
t.Errorf("placing the set-aside pipeline said %v, want its pipeline-started (A-446a)", said[saidBefore:])
}
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
@@ -278,16 +281,13 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) {
"batched": {Commit: "c2", Fp: "src2:bb", Repository: "o/batched"},
}}
view := func(string) (Half, bool) { return half, true }
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12,
ev := catalog.TakeInTaken{Branch: "main", Source: 9, Revision: 12,
Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}}
n, err := s.TakenIn(ev, view)
if err != nil || n != 1 {
t.Fatalf("the take-in started %d (%v), want its one entry", n, err)
}
if _, _, err := mem.Get("main.9.moved"); err != nil {
t.Errorf("main.9.moved is not started: %v", err)
}
if _, _, err := mem.Get("main.9.batched"); err == nil {
t.Errorf("main.9.batched is started, but the take-in does not name it")
if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
}
}
@@ -16,6 +16,8 @@ import (
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
)
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
@@ -137,8 +139,8 @@ type Hearer struct {
Ager *Ager
// Fills says the stores' fill at each step's end (§366); nil says none.
Fills *Fills
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
Widens func(seat.Pipeline) []seat.Widening
// Widens computes a mover's widens, kept in its value at the pipeline's end (§71, A-207; H-A-51); nil keeps none.
Widens func(p seat.Pipeline, module string) []seat.Widening
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
TestRuns TestRuns
@@ -198,14 +200,18 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
}
i := stepOfAsk(p, ev.ID)
var m seat.Mover
if i >= 0 {
m, _ = MoverOf(p, p.Steps[i].Module)
}
switch {
case i < 0:
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
case p.Steps[i].Seat != bs.Seat:
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
case !open(p.Steps[i]):
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
@@ -229,7 +235,12 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if ev.Output != "" && h.Output != nil {
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact))
// A mover's build step names no artifact: its output is kept under its module, so two movers' never meet (§746).
art := s.Artifact
if art == "" {
art = s.Module
}
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(art))
if err := h.Output(name, []byte(ev.Output)); err != nil {
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
}
@@ -251,7 +262,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
if err != nil {
return p, err
}
ended := endPipeline(&p, now)
ended := h.endPipeline(&p, now)
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
@@ -366,10 +377,11 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
if err := h.say(seat.EventWaitHeld, p, nil); err != nil {
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall;
// an end is said once, on <branch>.<source> (design 47 H §746). The runner says no group-ended (§764).
func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
for _, i := range asked {
if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil {
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
}
}
@@ -377,49 +389,6 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
return h.groupEnded(p)
}
return nil
}
// groupEnded says a group's one pipeline's end once every member's pipeline of the same watched branch and source revision
// ended (design 47 G §73, §332): on the subject under the group, no module part, with every member's end. A pipeline of no
// group says none; a member still running says none yet.
func (h *Hearer) groupEnded(p seat.Pipeline) error {
if p.Group == "" {
return nil
}
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return fmt.Errorf("the members of group %s are not listed: %w", p.Group, err)
}
sort.Strings(pkeys)
var members []seat.PipelineEnded
outcome, why := seat.GroupEndedOutcomePassed, ""
var last *time.Time
for _, key := range pkeys {
m, _, err := h.Store.Get(key)
if err != nil || m.Branch != p.Branch || m.Source != p.Source || m.Group != p.Group {
continue
}
if m.Outcome == seat.PipelineOutcomeRunning {
return nil
}
members = append(members, pipelineEnded(m))
if m.Ended != nil && (last == nil || m.Ended.After(*last)) {
last = m.Ended
}
if m.Outcome != seat.PipelineOutcomePassed && outcome != seat.GroupEndedOutcomeFailed {
outcome, why = m.Outcome, m.Module+": "+m.Why
if m.Outcome == seat.PipelineOutcomeFailed {
outcome = seat.GroupEndedOutcomeFailed
}
}
}
subject := "mesh.seat." + seat.Seat + ".event." + seat.EventGroupEnded + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(p.Group)
if err := h.Say(subject, seat.GroupEnded{V: 1, Branch: p.Branch, Group: p.Group, Source: p.Source, Revision: p.Revision, Ended: last,
Outcome: outcome, Why: why, Pipelines: members}); err != nil {
return fmt.Errorf("group %s ended but %s is not said: %w", p.Group, seat.EventGroupEnded, err)
}
return nil
}
@@ -436,37 +405,31 @@ func (h *Hearer) logf(format string, args ...any) {
}
}
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries.
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step event or a wait
// carries. A step's events are said on <module>.<branch>.<commit> of its mover; a pipeline's end and its waits on
// <branch>.<source> (design 47 H §746, H-A-72 (3), (4)).
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
last := p.Commit
if p.Group != "" {
last = p.Group
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
_, ev := stepEvent(event, p, *step, at)
subject, ev := stepEvent(event, p, *step, at)
return h.Say(subject, ev)
case seat.EventPipelineEnded:
e := pipelineEnded(p)
if h.Widens != nil {
e.Widens = h.Widens(p)
}
return h.Say(subject, e)
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p))
default:
return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at})
w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run,
Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at}
if step != nil {
m, _ := MoverOf(p, step.Module)
w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask
}
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w)
}
}
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with
// its shared when it reads a run (§729; A-468e, A-469q).
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for its mover: its result as it stands,
// with its shared when it reads a run (§729; A-468e, A-469q; H §746).
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
last := p.Commit
if p.Group != "" {
last = p.Group
}
if event == seat.EventStepStarted {
// A step-started names the run and its ask, never the step that asked it (A-469q).
var shared []seat.Shared
@@ -476,9 +439,10 @@ func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time
}
step.Shared = shared
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step}
m, _ := MoverOf(p, step.Module)
return runnerclient.Subject(event, m.Module, p.Branch, m.Commit), seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline,
Branch: p.Branch, Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint, Revision: p.Revision, Source: p.Source,
Run: p.Run, Result: step}
}
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
@@ -510,12 +474,28 @@ func testCounts(steps []seat.StepResult) *seat.TestCounts {
return &t
}
// stepsOf are the steps of p's mover module.
func stepsOf(p seat.Pipeline, module string) []seat.StepResult {
out := []seat.StepResult{}
for _, s := range p.Steps {
if s.Module == module {
out = append(out, s)
}
}
return out
}
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per
// mover, in the delivery order of its tiers, its module, fingerprint, released version, step results and tier.
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
tests := testCounts(p.Steps)
return seat.PipelineEnded{Tests: tests, OverBudget: tests != nil && (tests.Unit > budgetUnit || tests.Integration > budgetIntegration),
V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started,
Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps}
movers := make([]seat.MoverEnded, 0, len(p.Movers))
for _, m := range p.Movers {
movers = append(movers, seat.MoverEnded{Module: m.Module, Fingerprint: m.Fingerprint, Version: m.Version,
Steps: stepsOf(p, m.Module), Tier: m.Tier})
}
sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier })
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision,
Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers}
}
func endOutcome(o string) bool {
@@ -536,20 +516,32 @@ func stepOfAsk(p seat.Pipeline, id string) int {
return -1
}
// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else
// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step
// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the
// pipeline ended now, its end count one higher.
// stepPassed says a required step holds nothing: it passed, was carried, or had nothing to run (design 47 H §746, ADR
// 0682 §8, H-A-3).
func stepPassed(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomePassed || s.Outcome == seat.StepResultOutcomeCarried || s.Outcome == seat.StepResultOutcomeNothingToRun
}
// endPipeline ends a pipeline whose every mover is placed and every step ended: passed when every mover's required steps
// passed, were carried or had nothing to run, else failed, naming each required step that held it (ADR 0666 §7, §9;
// design 47 H §746). A failed optional step, and an optional step still waiting, hold nothing. An approval step still
// waiting when a required step did not pass is `not asked` (§26). It says whether the pipeline ended now, its end count one
// higher.
func endPipeline(p *seat.Pipeline, now time.Time) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
for _, m := range p.Movers {
if !placed(*p, m.Module) {
return false
}
}
var held []string
notAsked := -1
var notAsked []int
for i, s := range p.Steps {
switch {
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
notAsked = i
notAsked = append(notAsked, i)
continue
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
@@ -558,15 +550,17 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome)
if s.Status == seat.StepResultStatusRequired && !stepPassed(s) {
held = append(held, strings.TrimSpace(s.Module+" "+s.Artifact+" "+s.Step)+" "+s.Outcome)
}
}
if notAsked >= 0 {
if len(notAsked) > 0 {
if len(held) == 0 {
return false
}
p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked
for _, i := range notAsked {
p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked
}
}
p.Ended = &now
p.EndCount++
@@ -574,6 +568,24 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
if len(held) > 0 {
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
}
for i := range p.Movers {
m := &p.Movers[i]
m.Tests = testCounts(stepsOf(*p, m.Module))
m.OverBudget = m.Tests != nil && (m.Tests.Unit > budgetUnit || m.Tests.Integration > budgetIntegration)
}
return true
}
// endPipeline is endPipeline with each mover's widens computed into its value at the end (§71, A-207; H-A-51).
func (h *Hearer) endPipeline(p *seat.Pipeline, now time.Time) bool {
if !endPipeline(p, now) {
return false
}
if h.Widens != nil {
for i := range p.Movers {
p.Movers[i].Widens = h.Widens(*p, p.Movers[i].Module)
}
}
return true
}
@@ -590,7 +602,7 @@ func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
if len(steps) > 0 {
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
}
if err := h.said(p, false, true); err != nil && first == nil {
if err := h.said(p, nil, true); err != nil && first == nil {
first = err
}
return first
@@ -67,17 +67,16 @@ type askRecorder struct{ asked []oc.Ask }
func (a *askRecorder) Ask(x oc.Ask) error { a.asked = append(a.asked, x); return nil }
const pkey = "main.5.app"
const pkey = "main.5"
// fixture is a pipeline of app with a required lint asked of node-build-go, an optional unit asked of the same seat and an
// approval step waiting, and a hearer over it.
func fixture() (*Hearer, memStore, *[]string, *askRecorder, map[string]seat.KeptResult, map[string]string) {
store := memStore{pkey: {p: seat.Pipeline{V: 1, Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa",
store := memStore{pkey: {p: seat.Pipeline{V: 1, Pipeline: pkey, Branch: "main", Source: 5,
Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{
{Artifact: "program", Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "a1", Seat: nodebuildgo.Seat},
{Artifact: "program", Step: seat.StepResultStepUnit, Status: seat.StepResultStatusOptional, Outcome: seat.StepResultOutcomeRunning, Ask: "a2", Seat: nodebuildgo.Seat},
{Artifact: "program", Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeWaiting},
}}, rev: 1}}
{Module: "app", Artifact: "program", Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "a1", Seat: nodebuildgo.Seat},
{Module: "app", Artifact: "program", Step: seat.StepResultStepUnit, Status: seat.StepResultStatusOptional, Outcome: seat.StepResultOutcomeRunning, Ask: "a2", Seat: nodebuildgo.Seat},
{Module: "app", Artifact: "program", Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeWaiting}}, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, rev: 1}}
said := &[]string{}
asker := &askRecorder{}
kept := map[string]seat.KeptResult{}
@@ -108,7 +107,7 @@ func TestStepEndedIsTakenOnlyAsAsked(t *testing.T) {
wrongNode := ended("a1", "lint", "passed")
wrongNode.Node = "n2"
unknownPipeline := ended("a1", "lint", "passed")
unknownPipeline.Pipeline = "main.9.app"
unknownPipeline.Pipeline = "main.9"
for name, c := range map[string]struct {
subject string
ev StepEnded
@@ -194,7 +193,7 @@ func TestApprovalIsAskedLastAndEndedByItsWarrant(t *testing.T) {
t.Fatalf("asked %+v", asker.asked)
}
a := asker.asked[0]
if got := (*said)[len(*said)-1]; !strings.HasPrefix(got, "wait-held.app.main.c1") {
if got := (*said)[len(*said)-1]; !strings.HasPrefix(got, "wait-held.main.5") {
t.Errorf("the wait was said as %q", got)
}
warrant := func(option, outcome string) oc.Warrant {
@@ -240,7 +239,7 @@ func TestApprovalIsAskedLastAndEndedByItsWarrant(t *testing.T) {
if err != nil || p.Steps[2].Outcome != seat.StepResultOutcomePassed || p.Outcome != seat.PipelineOutcomePassed || p.EndCount != 1 {
t.Fatalf("approved: %+v (%v)", p, err)
}
if !strings.HasPrefix((*said)[len(*said)-1], "pipeline-ended.app.main.c1") {
if !strings.HasPrefix((*said)[len(*said)-1], "pipeline-ended.main.5") {
t.Errorf("said %v", *said)
}
// A pipeline that ended takes no second decision.
@@ -326,7 +325,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
var to []string
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want {
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
}
}
@@ -4,8 +4,6 @@ import (
"errors"
"fmt"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder
@@ -19,7 +17,7 @@ func (h *Hearer) Taken(ask, node string) error {
if ask == "" || node == "" {
return nil
}
pkeys, err := h.Store.Keys("*.*.*")
pkeys, err := allKeys(h.Store)
if err != nil {
return fmt.Errorf("the pipelines are not listed: %w", err)
}
@@ -45,113 +43,4 @@ func (h *Hearer) Taken(ask, node string) error {
return nil
}
// Supersede ends, for the pipeline newer just written, each older running pipeline of the same watched branch and module
// whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started
// (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended
// `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves
// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
// request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended.
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) {
if newer.Run != "" || newer.Source <= 0 {
return 0, nil
}
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
why := supersededWhy(newer)
ended := 0
var first error
for _, key := range pkeys {
if key == newer.Pipeline {
continue
}
done, err := h.supersedeOne(key, newer, why, drop)
if err != nil && first == nil {
first = err
}
if done {
ended++
}
}
return ended, first
}
// superseded says whether old is a running pipeline newer supersedes (§31, A-202): of the same watched branch and an older
// source revision, either of the same module at another fingerprint, or, for newer a package's publish started, one an older
// publish of that package started (its version another, made from an older source revision of the branch).
func superseded(old, newer seat.Pipeline) bool {
if old.Outcome != seat.PipelineOutcomeRunning || old.Run != "" || old.Branch != newer.Branch || old.Source >= newer.Source {
return false
}
if newer.Published != nil && old.Published != nil && old.Published.Artifact == newer.Published.Artifact {
return old.Published.Version != newer.Published.Version
}
return old.Module == newer.Module && old.Fingerprint != newer.Fingerprint
}
// supersededWhy names the newer commit, or the newer publish, that ended a pipeline.
func supersededWhy(newer seat.Pipeline) string {
if newer.Published != nil {
return fmt.Sprintf("the newer publish %s %s of %s supersedes it", newer.Published.Artifact, newer.Published.Version, newer.Branch)
}
return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module)
}
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) {
for {
p, rev, err := h.Store.Get(key)
if err != nil || !superseded(p, newer) {
return false, nil
}
now := h.now()
var drops []seat.StepResult
var changed []int
for i := range p.Steps {
s := &p.Steps[i]
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
drops = append(drops, *s)
}
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
changed = append(changed, i)
}
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, why, ptrTime(now)
p.EndCount++
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
var first error
for _, i := range changed {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
}
}
if len(changed) > 0 {
h.appendUnlogged(key)
}
for _, i := range changed {
h.leaveRun(key, p.Steps[i], why)
}
for _, s := range drops {
if drop != nil {
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err)
}
}
}
if err := h.said(p, false, true); err != nil && first == nil {
first = err
}
return true, first
}
}
func ptrTime(t time.Time) *time.Time { return &t }
@@ -2,93 +2,21 @@ package pipeline
import (
"errors"
"slices"
"strings"
"testing"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
)
// RN8 (design 47 G §31, §119, §350; A-182, A-202): a step keeps the node that took its ask; a newer commit changing the
// module's fingerprint, or a newer publish of the package that started a pipeline, ends the older running pipeline
// superseded, its asks dropped of the node that took each (any holder for one never taken), a mesh-builds step only ended;
// an unchanged fingerprint, a run on request and an equal publish supersede nothing.
func TestSupersede(t *testing.T) {
h, store, said, _, _, _ := fixture()
old := store[pkey]
old.p.Steps = append(old.p.Steps, seat.StepResult{Artifact: "program", Step: seat.StepResultStepComposition, Status: seat.StepResultStatusRequired,
Outcome: seat.StepResultOutcomeRunning, Ask: "b1", Seat: builds.Seat})
store[pkey] = old
// RN8 (design 47 G §31, §119): a step keeps the node that took its ask. A running pipeline runs to its end: a newer one
// supersedes it no more (design 47 H §748).
func TestATakenAskNamesItsNode(t *testing.T) {
h, store, _, _, _, _ := fixture()
if err := h.Taken("a1", "n7"); err != nil {
t.Fatal(err)
}
if p, _, _ := store.Get(pkey); p.Steps[0].Node != "n7" || p.Steps[1].Node != "" {
t.Fatalf("after ask-taken of a1 on n7 the steps name %q and %q", p.Steps[0].Node, p.Steps[1].Node)
}
type dropped struct{ seat, node, ask string }
var drops []dropped
drop := func(s, node, ask, why string) error {
if !strings.Contains(why, "c2") {
t.Errorf("the drop of %s names no newer commit: %q", ask, why)
}
drops = append(drops, dropped{s, node, ask})
return nil
}
newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline {
return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub,
Outcome: seat.PipelineOutcomeRunning}
}
for _, c := range []struct {
name string
newer seat.Pipeline
}{
{"an unchanged fingerprint", newer("main.6.app", 6, "src2:aa", "", nil)},
{"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)},
{"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)},
} {
if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil {
t.Errorf("%s superseded %d (%v)", c.name, n, err)
}
}
*said = nil
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil {
t.Fatalf("a changed fingerprint superseded %d (%v)", n, err)
}
p, _, _ := store.Get(pkey)
if p.Outcome != seat.PipelineOutcomeSuperseded || p.EndCount != 1 || !strings.Contains(p.Why, "c2") {
t.Errorf("the older pipeline is %s (%d ends): %q", p.Outcome, p.EndCount, p.Why)
}
for _, s := range p.Steps {
if s.Outcome != seat.StepResultOutcomeSuperseded {
t.Errorf("the %s step is %s, want superseded", s.Step, s.Outcome)
}
}
want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}}
if !slices.Equal(drops, want) {
t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want)
}
if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" {
t.Errorf("said %v, want four step-ended then pipeline-ended", *said)
}
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 {
t.Errorf("a superseded pipeline was superseded again: %d", n)
}
// A newer publish of a package supersedes the pipelines an older publish of it started, whatever their module.
pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa",
Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}}
h.Store = pub
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 {
t.Errorf("a publish of the same version superseded %d", n)
}
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil {
t.Fatalf("a newer publish superseded %d (%v)", n, err)
}
if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") {
t.Errorf("the older publish's pipeline is %s: %q", p.Outcome, p.Why)
}
}
// A-222, A-77: a drop is bounded by a key of its own armed before it is asked and deleted when it is answered; one never
@@ -128,7 +128,7 @@ func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
if store == nil || ask == "" {
return nil, nil
}
pkeys, err := store.Keys("*.*.*")
pkeys, err := allKeys(store)
if err != nil {
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
}
@@ -113,10 +113,10 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := f.store.Get(Key("main", source, "app"))
p, _, _ := f.store.Get(Key("main", source))
return p
}
@@ -308,15 +308,16 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
if saidReaders != 4 {
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
}
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
// a, cancelled, leaves the lint run to b's readers; b, cancelled after, leaves none: the ask is dropped and the run ends.
lint := stepNamed(a, "app:program", "lint")
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil {
v := Verbs{Hearer: f.h, Acts: &actLog{}}
if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: a.Pipeline, Why: "not needed"}); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
}
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil {
if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: b.Pipeline, Why: "not needed"}); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
@@ -327,7 +328,7 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
p := c.start(t, 4, 9)
comp := stepNamed(p, "", "composition")
bs := stepSeatOf(t, "mesh-wants")
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
ev, err := bs.Decode(raw)
if err != nil {
@@ -538,7 +539,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.DropRun = func(string, string, string, string) error {
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
Readers: []seat.TestRunReader{{Pipeline: "main.5", Artifact: "app:program"}}})
return nil
}
for _, a := range []string{"app:program", "app:tools"} {
@@ -648,7 +649,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
}
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
r, rev, _ := f.runs.Get(run)
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9", Artifact: "app:program"})
_, _ = f.runs.Update(run, r, rev)
}}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
@@ -663,7 +664,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
r, _, _ := f.runs.Get(run)
joined := false
for _, o := range r.Readers {
joined = joined || o.Pipeline == "main.9.app"
joined = joined || o.Pipeline == "main.9"
}
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
@@ -71,7 +71,7 @@ func (h *Hearer) appendUnlogged(key string, readBack ...bool) int {
// AppendUnlogged appends the unlogged entries of every pipeline, each after reading back the log's last entry of its pipeline (§549):
// at the runner's start, and each minute while the step log is full (§606). It says how many it appended.
func (h *Hearer) AppendUnlogged() int {
pkeys, err := h.Store.Keys("*.*.*")
pkeys, err := allKeys(h.Store)
if err != nil {
h.logf("the pipelines are not listed, so no unlogged entry is appended: %v", err)
return 0
@@ -73,7 +73,7 @@ func (v Verbs) change(key, event string, record func() error, mutate func(p *sea
if err != nil {
return p, i, err
}
ended := endPipeline(&p, now)
ended := h.endPipeline(&p, now)
if i >= 0 && event == seat.EventStepEnded {
p.Unlogged = append(p.Unlogged, endEntry(key, p.Steps[i], now))
}
@@ -101,18 +101,21 @@ func (v Verbs) Review(caller string, in seat.ReviewInput) (seat.ReviewAnswer, er
if in.Outcome != seat.ReviewInputOutcomePassed && in.Outcome != seat.ReviewInputOutcomeFailed {
return seat.ReviewAnswer{}, fmt.Errorf("a review ends %q or %q, not %q", seat.ReviewInputOutcomePassed, seat.ReviewInputOutcomeFailed, in.Outcome)
}
keys, err := v.Hearer.Store.Keys("*.*.*")
keys, err := allKeys(v.Hearer.Store)
if err != nil {
return seat.ReviewAnswer{}, fmt.Errorf("the pipelines are not listed: %w", err)
}
var answer *seat.ReviewAnswer
for _, key := range keys {
p, _, err := v.Hearer.Store.Get(key)
if err != nil || p.Module != in.Module || p.Commit != in.Commit || p.Fingerprint != in.Fingerprint || reviewWaiting(p) < 0 {
if err != nil {
continue
}
if m, ok := MoverOf(p, in.Module); !ok || m.Commit != in.Commit || m.Fingerprint != in.Fingerprint || reviewWaiting(p, in.Module) < 0 {
continue
}
written, i, err := v.change(key, seat.EventStepEnded, nil, func(p *seat.Pipeline, now time.Time) (int, error) {
i := reviewWaiting(*p)
i := reviewWaiting(*p, in.Module)
if i < 0 {
return -1, fmt.Errorf("%s has no review step waiting any more", key)
}
@@ -133,12 +136,12 @@ func (v Verbs) Review(caller string, in seat.ReviewInput) (seat.ReviewAnswer, er
return *answer, nil
}
func reviewWaiting(p seat.Pipeline) int {
func reviewWaiting(p seat.Pipeline, module string) int {
if p.Outcome != seat.PipelineOutcomeRunning {
return -1
}
for i, s := range p.Steps {
if s.Step == seat.StepResultStepReview && s.Outcome == seat.StepResultOutcomeWaiting {
if s.Module == module && s.Step == seat.StepResultStepReview && s.Outcome == seat.StepResultOutcomeWaiting {
return i
}
}
@@ -226,7 +229,7 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
for _, s := range leave {
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
}
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
return seat.CancelAnswer{Pipeline: p}, h.said(p, nil, true)
}
}
@@ -265,7 +268,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
return -1, err
}
}
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
*s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true}
if ask != nil {
s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID
@@ -289,7 +292,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
}
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
@@ -353,7 +356,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
return -1, err
}
s := &p.Steps[i]
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
*s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now}
switch {
case ask != nil:
@@ -436,7 +439,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
break
}
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
@@ -473,9 +476,13 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") {
return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask")
}
reg, ok := v.Starter.Registration(p.Module)
m, ok := MoverOf(p, s.Module)
if !ok {
return nil, fmt.Errorf("%s has no registration", p.Module)
return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module)
}
reg, ok := v.Starter.Registration(m.Module)
if !ok {
return nil, fmt.Errorf("%s has no registration", m.Module)
}
var half Half
if p.Run != "" {
@@ -491,13 +498,13 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
} else {
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
}
_, placed := place(p, reg, half.Entries, now, v.Starter.Name, false)
base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step)
_, placed := place(p, m, reg, half.Entries, now, v.Starter.Name, false)
base := AskID(startOf(p, now), p.Pipeline, m.Module, s.Artifact, s.Step)
for _, a := range placed {
if a.ID != base {
continue
}
a.ID = AskID(now, p.Pipeline, s.Artifact, s.Step)
a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step)
switch b := a.Ask.(type) {
case nodebuildgo.StepAsk:
b.ID = a.ID
@@ -525,7 +532,7 @@ func (v Verbs) Run(caller string, in seat.RunInput) (seat.RunAnswer, error) {
if v.Starter == nil {
return seat.RunAnswer{}, errors.New("the runner cannot start a run: it holds no starter")
}
return v.Starter.Run(caller, in, NewRunID)
return v.Starter.Run(caller, in, NewRunID, v.View)
}
// Start starts by hand a delivering pipeline for the head of a watched branch the runner's catalog half holds, recorded in
@@ -546,12 +553,13 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
if !ok || half.Source <= 0 {
return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module)
}
key := Key(in.Branch, half.Source, in.Module)
key := Key(in.Branch, half.Source)
if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil {
return seat.StartAnswer{}, err
}
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Module: in.Module, Commit: e.Commit, Fp: e.Fp,
Repository: e.Repository, Revision: half.Revision, Entries: half.Entries})
// A hand start keys the newest revision's <branch>.<source> with the module as its one mover (H-A-52).
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}})
if err != nil {
return seat.StartAnswer{}, err
}
@@ -580,7 +588,7 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw
if s.Outcome != seat.StepResultOutcomeDeclined {
return -1, fmt.Errorf("only a declined approval is asked again: the approval of %s is %s", p.Pipeline, s.Outcome)
}
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint,
*s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint,
Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true, Started: &now}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
@@ -592,3 +600,16 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw
}
return seat.AskAgainAnswer{Result: p.Steps[i]}, nil
}
// Place places a waiting pipeline ahead of its turn, written in the hand-act log (design 47 H §748, H-A-53): only a
// pipeline waiting under pipelines-at-once is placed; any other is refused, nothing done.
func (v Verbs) Place(caller string, in seat.PlaceInput) (seat.PlaceAnswer, error) {
p, _, err := v.Hearer.Store.Get(in.Pipeline)
if err != nil {
return seat.PlaceAnswer{}, fmt.Errorf("the pipeline %s is not read: %w", in.Pipeline, err)
}
if p.Outcome != seat.PipelineOutcomeWaiting {
return seat.PlaceAnswer{}, fmt.Errorf("the pipeline %s is %s, not waiting: nothing was placed", in.Pipeline, p.Outcome)
}
return seat.PlaceAnswer{}, fmt.Errorf("the pipeline %s waits, and the runner holds no queue to place it from: nothing was placed", in.Pipeline)
}
@@ -25,9 +25,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string)
reg, entries := placeFixture()
reg.Policy.Steps.Review = "optional"
h, store, said, _, _, _ := fixture()
p := seat.Pipeline{Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app",
Outcome: seat.PipelineOutcomeRunning}
p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner")
p := seat.Pipeline{Pipeline: pkey, Branch: "main", Source: 5,
Outcome: seat.PipelineOutcomeRunning, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}
p.Steps, _ = Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner")
store[pkey] = struct {
p seat.Pipeline
rev uint64
@@ -236,7 +236,7 @@ func TestStartAndAskAgain(t *testing.T) {
_, entries := placeFixture()
v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true }
a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"})
if err != nil || a.Pipeline != "main.7.app" || made["main.7.app"] == nil {
if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil {
t.Fatalf("start = %+v, %v; written %v", a, err, len(made))
}
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil {
@@ -253,7 +253,7 @@ func TestStartAndAskAgain(t *testing.T) {
h := v.Hearer
asker := h.Asker.(*askRecorder)
e := store[pkey]
first, err := ApprovalAsk(e.p, time.Unix(1, 0))
first, err := ApprovalAsk(e.p, "app", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
@@ -282,7 +282,7 @@ func TestStartAndAskAgain(t *testing.T) {
t.Errorf("the pipeline is %s after ask-again, want running", store[pkey].p.Outcome)
}
// The ask rebuilt from the step, as a warrant is held to it, is the ask asked.
if again, err := ApprovalAsk(store[pkey].p, *stepNamed(store[pkey].p, "", seat.StepResultStepApproval).Started); err != nil || again.ID != got.Result.Ask {
if again, err := ApprovalAsk(store[pkey].p, "app", *stepNamed(store[pkey].p, "", seat.StepResultStepApproval).Started); err != nil || again.ID != got.Result.Ask {
t.Errorf("the ask rebuilt = %s, %v; want %s", again.ID, err, got.Result.Ask)
}
if _, err := v.AskAgain("operator", seat.AskAgainInput{Pipeline: pkey, Why: "twice"}); err == nil {
@@ -3,7 +3,6 @@ package pipeline
import (
"fmt"
"sort"
"time"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -28,7 +27,7 @@ type Widener struct {
// Widens names each part serving a step the change alters (the holders of the runner's seat, the catalog's, each of the pipeline's
// steps' and the composer's, whose entry's fingerprint moved), each acting rule of the pipeline's module that is gained, matches
// more or is held longer, each own secret or credential provision it names that it did not, and each grant the grants part adds.
func (w *Widener) Widens(p seat.Pipeline) []seat.Widening {
func (w *Widener) Widens(p seat.Pipeline, module string) []seat.Widening {
if w == nil || w.Revision == nil {
return nil
}
@@ -56,8 +55,8 @@ func (w *Widener) Widens(p seat.Pipeline) []seat.Widening {
Said: fmt.Sprintf("serves %s; its fingerprint is %s, was %s", s, after.Fp, before.Fp)})
}
}
out = append(out, ruleWidens(p.Module, newest.Entries[p.Module], cand.Entries[p.Module])...)
out = append(out, credentialWidens(p.Module, newest.Entries[p.Module], cand.Entries[p.Module])...)
out = append(out, ruleWidens(module, newest.Entries[module], cand.Entries[module])...)
out = append(out, credentialWidens(module, newest.Entries[module], cand.Entries[module])...)
if w.GrantsFor != nil {
g, err := w.GrantsFor(p.Branch, p.Source)
switch {
@@ -67,7 +66,7 @@ func (w *Widener) Widens(p seat.Pipeline) []seat.Widening {
out = append(out, seat.Widening{What: seat.WideningWhatCouldNotRun, Name: "mesh-grants", Said: g.Error})
default:
for _, m := range g.Modules {
if m.Module != p.Module {
if m.Module != module {
continue
}
for _, a := range m.Added {
@@ -105,7 +104,7 @@ func servingSeats(p seat.Pipeline) []string {
}
// ruleWidens names each acting rule of the candidate's entry that the newest entry has not, or that now names more entries
// (`every` where it named a language) or holds longer (A-215).
// (`every` where it named a language) (A-215); an acting rule names no bound (H-A-70).
func ruleWidens(module string, before, after catalog.CatalogEntry) []seat.Widening {
on := func(s string) string {
if s == "" {
@@ -127,19 +126,11 @@ func ruleWidens(module string, before, after catalog.CatalogEntry) []seat.Wideni
out = append(out, seat.Widening{What: seat.WideningWhatActingRule, Name: name, Said: "the rule is gained"})
case on(r.On) == "every" && on(was.On) != "every":
out = append(out, seat.Widening{What: seat.WideningWhatActingRule, Name: name, Said: "the rule acts on every entry, it acted on " + was.On})
case longer(r.Bound, was.Bound):
out = append(out, seat.Widening{What: seat.WideningWhatActingRule, Name: name, Said: "the bound is " + r.Bound + ", it was " + was.Bound})
}
}
return out
}
func longer(a, b string) bool {
x, err1 := time.ParseDuration(a)
y, err2 := time.ParseDuration(b)
return err1 == nil && err2 == nil && x > y
}
// credentialWidens names each own secret and each provision with a credential of its own that the candidate's manifest names and
// the newest entry's did not (A-215).
func credentialWidens(module string, before, after catalog.CatalogEntry) []seat.Widening {
@@ -4,6 +4,7 @@ import (
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -23,12 +24,12 @@ func halfOf(rev int64, es ...catalog.CatalogEntry) Half {
}
// widens compares the pipeline's own revision with the branch's newest: the serving parts found by the holder of the seats
// (the runner, the catalog, each step's seat, the node-engine) by fingerprint, an acting rule gained, matching more or held longer, a
// (the runner, the catalog, each step's seat, the node-engine) by fingerprint, an acting rule gained or matching more, a
// credential named that was not, and the grants the grants part adds; each is said by name, a grants part that does not answer says
// `could not run` and nothing is refused (design 47 G §71, §275, §441, A-207, A-215, A-220).
func TestWidensIsTheCandidateAgainstTheNewest(t *testing.T) {
p := seat.Pipeline{Pipeline: "main.8.app", Module: "app", Branch: "main", Source: 31, Revision: 8,
Steps: []seat.StepResult{{Step: "unit", Seat: "node-build-go"}, {Step: "lint", Seat: "node-build-go"}, {Step: "approval"}}}
p := seat.Pipeline{Pipeline: "main.8.app", Branch: "main", Source: 31, Revision: 8,
Steps: []seat.StepResult{{Module: "app", Step: "unit", Seat: "node-build-go"}, {Module: "app", Step: "lint", Seat: "node-build-go"}, {Module: "app", Step: "approval"}}, Movers: []seat.Mover{{Module: "app"}}}
holders := map[string]string{seat.Seat: "runner", catalog.Seat: "indexer", "node-build-go": "build-go", "node-engine": "host"}
rule := func(on, bound string) []catalog.ActingRule {
return []catalog.ActingRule{{Seat: "node-build-go", Step: "check", On: on, Bound: bound}}
@@ -50,9 +51,7 @@ func TestWidensIsTheCandidateAgainstTheNewest(t *testing.T) {
meshgrants.GrantsFor{}, nil, []string{"acting rule app x.grant"}},
{"an acting rule matches more", halfOf(8, entry("app", "fp-1", rule("every", "10m"), newest.Entries["app"].Manifest)), meshgrants.GrantsFor{}, nil,
[]string{"acting rule app node-build-go.check"}},
{"an acting rule is held longer", halfOf(8, entry("app", "fp-1", rule("go", "1h"), newest.Entries["app"].Manifest)), meshgrants.GrantsFor{}, nil,
[]string{"acting rule app node-build-go.check"}},
{"an acting rule is held shorter", halfOf(8, entry("app", "fp-1", rule("go", "5m"), newest.Entries["app"].Manifest)), meshgrants.GrantsFor{}, nil, nil},
{"an acting rule names no bound to widen (H-A-70)", halfOf(8, entry("app", "fp-1", rule("go", "1h"), newest.Entries["app"].Manifest)), meshgrants.GrantsFor{}, nil, nil},
{"a secret and a credential provision are named", halfOf(8, entry("app", "fp-1", rule("go", "10m"), map[string]any{
"secrets": map[string]any{"token": "/x", "key": map[string]any{"path": "/k"}},
"provides": []any{"db", map[string]any{"name": "vault", "credential": map[string]any{"own": "key"}}}})),
@@ -78,7 +77,7 @@ func TestWidensIsTheCandidateAgainstTheNewest(t *testing.T) {
GrantsFor: func(string, int64) (meshgrants.GrantsFor, error) { return c.grants, c.grantsErr },
}
var got []string
for _, x := range w.Widens(p) {
for _, x := range w.Widens(p, "app") {
got = append(got, x.What+" "+x.Name)
}
if strings.Join(got, ";") != strings.Join(c.want, ";") {
@@ -87,22 +86,19 @@ func TestWidensIsTheCandidateAgainstTheNewest(t *testing.T) {
}
}
// pipeline-ended carries the widens the runner computed, and says it whether or not anything widens (design 47 G §71, A-207).
func TestPipelineEndedCarriesWidens(t *testing.T) {
var body any
h := &Hearer{Say: func(_ string, b any) error { body = b; return nil },
Widens: func(p seat.Pipeline) []seat.Widening {
return []seat.Widening{{What: seat.WideningWhatGrant, Name: p.Module + " s v"}}
}}
if err := h.say(seat.EventPipelineEnded, seat.Pipeline{Module: "app", Outcome: seat.PipelineOutcomePassed}, nil); err != nil {
t.Fatal(err)
}
if e, ok := body.(seat.PipelineEnded); !ok || len(e.Widens) != 1 || e.Widens[0].Name != "app s v" {
t.Errorf("pipeline-ended says %+v", body)
// A pipeline's end keeps each mover's widens in its value, whether or not anything widens (design 47 G §71, A-207; H H-A-51).
func TestAnEndKeepsEachMoversWidens(t *testing.T) {
h := &Hearer{Widens: func(p seat.Pipeline, module string) []seat.Widening {
return []seat.Widening{{What: seat.WideningWhatGrant, Name: module + " s v"}}
}}
p := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Movers: []seat.Mover{{Module: "app"}, {Module: "lib"}},
Steps: []seat.StepResult{{Module: "app", Step: "build", Outcome: "passed"}, {Module: "lib", Step: "build", Outcome: "passed"}}}
if !h.endPipeline(&p, time.Unix(1, 0)) || len(p.Movers[0].Widens) != 1 || p.Movers[0].Widens[0].Name != "app s v" || p.Movers[1].Widens[0].Name != "lib s v" {
t.Errorf("the ended movers = %+v", p.Movers)
}
h.Widens = nil
_ = h.say(seat.EventPipelineEnded, seat.Pipeline{Module: "app"}, nil)
if e := body.(seat.PipelineEnded); len(e.Widens) != 0 {
t.Errorf("a runner with no widener said %v", e.Widens)
q := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{{Module: "app", Step: "build", Outcome: "passed"}}}
if !h.endPipeline(&q, time.Unix(1, 0)) || len(q.Movers[0].Widens) != 0 {
t.Errorf("a runner with no widener kept %v", q.Movers[0].Widens)
}
}
@@ -18,6 +18,7 @@
"rerun",
"accept",
"cancel",
"place",
"ask-again",
"test-runs",
"test-runs-drop"