Merge branch 'fix/47-G-shared-runs-runner' into fix/47-G-shared-runs

This commit is contained in:
2026-11-17 12:30:47 +01:00
38 changed files with 4023 additions and 271 deletions
@@ -46,7 +46,7 @@ func as[T any](args map[string]any) (T, error) {
return in, nil
}
// Tools are the runner's read verbs over its states pipelines and kept.
// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i).
func Tools(r pipeline.Reads) []stdio.Tool {
runs := map[string]func(map[string]any) (any, error){
seat.VerbShow: func(a map[string]any) (any, error) {
@@ -152,6 +152,7 @@ func main() {
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
@@ -0,0 +1,43 @@
package main
import (
"encoding/json"
"errors"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb
// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i).
func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{
{Name: Prefix + "_test_runs",
Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"},
"digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}},
Run: func(args map[string]any) (any, error) {
in, err := as[seat.TestRunsInput](args)
if err != nil {
return nil, err
}
return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in)
}},
{Name: Prefix + "_test_runs_drop",
Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}},
Run: func(args map[string]any) (any, error) {
in, err := as[seat.TestRunsDropInput](args)
if err != nil {
return nil, err
}
if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" {
return nil, errors.New("test-runs-drop names the run's key and why")
}
return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in)
}},
}
}
@@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string
// armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222).
func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error {
return func(key string) error {
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound))
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
@@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask),
Headline: "A call whose answer is kept is not answered",
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound),
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound),
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}})
}
}
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no
// holder took (§32, §535, A-77); another key is none of the runner's.
func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
return func(key string) {
if s, ask, ok := pipeline.ParseDropBound(key); ok {
if err := unanswered(s, ask); err != nil {
logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err)
}
return
}
if !pipeline.IsAskKey(key) {
return
}
if err := h.NotTaken(key, pipeline.AskBound); err != nil {
logf("the step of ask %s, which no holder took, is not ended: %v", key, err)
}
}
}
@@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error
func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
w := &fakeFindings{}
var logged int
expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
expired("not-a-bound")
if len(w.written) != 0 {
t.Fatalf("a key of no drop was said: %+v", w.written)
@@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
if err != nil {
return err
}
@@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// for its step.
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
return err
}
resume := func(branch string) {
rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647)
half, ok := rv.Catalog.View(branch)
if !ok {
return
@@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
resumer.Store(&resume)
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
advance := func(branch string) {
rv.Holders.refresh()
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
@@ -227,23 +229,41 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31,
// §350; A-182, A-202).
starter.Supersede = func(newer seat.Pipeline) {
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds))
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)))
if err != nil {
logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err)
} else if n > 0 {
logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n)
}
}
// The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock.
runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))
runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf)
if err != nil {
return err
}
if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil {
return err
}
if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil {
return err
}
if err := hearStepEnded(heard, hearer, logger.Printf); err != nil {
return err
}
if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil {
return err
}
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
taken := func(ask, node string) error {
if err := hearer.Taken(ask, node); err != nil {
return err
}
return hearer.RunTaken(ask, node)
}
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
return err
}
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
return err
}
@@ -261,6 +281,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// An entry whose step ended and was not appended is appended at the start, in order (§606, §549).
hearer.AppendUnlogged()
// A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725).
if n, err := hearer.TakeUp(); err != nil || n > 0 {
logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err)
}
go whileTheLogIsFull(ctx, fills, hearer, time.Minute)
// The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check
// their callers against (A-186): registered before the hub starts, read again once its events are live.
@@ -273,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds))
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
stopVerbs, err := answerVerbs(calls, seats, verbs)
if err != nil {
return err
}
@@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
t.Fatal(err)
}
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
_ = m.Term()
return
}
_, err = h.OnStepEnded(bs, m.Subject(), ev)
var shared []seat.Shared
if bs.Shared != nil {
shared = bs.Shared(m.Data())
}
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
@@ -133,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
return nil
}
// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took
// it (A-182): the ask is answered, so its five minutes end with no finding.
func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error {
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
// that node's holder, and begins the clock of a test run's ask (§725).
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
for _, ts := range pipeline.TakenSeats {
ts := ts
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
var ev nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
@@ -145,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
_ = m.Term()
return
}
if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err)
_ = m.Nak()
return
}
// The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182).
if err := taken(ev.Ask, ev.Node); err != nil {
logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err)
_ = m.Nak()
@@ -165,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
return nil
}
// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask.
func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
return func(ask string) error {
_, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return err
}
}
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
// unanswered (A-222, A-77): no timer in the process.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
@@ -0,0 +1,416 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section.
const TestRunsState = "test-runs"
// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own
// time to live (A-77, A-470).
type testRunsKV struct {
ctx context.Context
js jetstream.JetStream
kv jetstream.KeyValue
}
func (t testRunsKV) Create(key string, r pipeline.Run) error {
raw, err := json.Marshal(r)
if err != nil {
return err
}
if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
} else if err != nil {
return err
}
return nil
}
func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) {
e, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return pipeline.Run{}, 0, pipeline.ErrNoRun
}
if err != nil {
return pipeline.Run{}, 0, err
}
var r pipeline.Run
if err := json.Unmarshal(e.Value(), &r); err != nil {
return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err)
}
return r, e.Revision(), nil
}
func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) {
raw, err := json.Marshal(r)
if err != nil {
return 0, err
}
rev, err := t.kv.Update(t.ctx, key, raw, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (t testRunsKV) Delete(key string, revision uint64) error {
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return pipeline.ErrChanged
}
if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// deleteBound deletes a bound key, whose value is no run: a key gone is no fault.
func (t testRunsKV) deleteBound(key string) error {
if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
type boundValue struct {
Bound string `json:"bound"`
Ask string `json:"ask"`
}
// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
// takes its time to live with it, so the expiry is heard only ttl after the last write.
func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) {
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
if err != nil {
return 0, err
}
subject := "$KV." + t.kv.Bucket() + "." + key
for {
var last uint64
st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket())
if err != nil {
return 0, err
}
m, err := st.GetLastMsgForSubject(t.ctx, subject)
switch {
case err == nil:
last = m.Sequence
case !errors.Is(err, jetstream.ErrMsgNotFound):
return 0, err
}
ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
continue
}
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
}
// Bounded reads a bound key: the ask it holds and its revision.
func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) {
e, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return "", 0, false, nil
}
if err != nil {
return "", 0, false, err
}
var v boundValue
_ = json.Unmarshal(e.Value(), &v)
return v.Ask, e.Revision(), true, nil
}
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
// UnboundAt deletes a bound key at revision: a later write of it is kept.
func (t testRunsKV) UnboundAt(key string, revision uint64) error {
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) {
return nil
}
return err
}
func (t testRunsKV) Has(key string) (bool, error) {
_, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return false, nil
}
return err == nil, err
}
func (t testRunsKV) Keys() ([]string, error) {
all, err := t.kv.Keys(t.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return all, err
}
// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written.
type runsKept struct {
ctx context.Context
kv jetstream.KeyValue
}
func (r runsKept) Keys() ([]string, error) {
ks, err := r.kv.Keys(r.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return ks, err
}
func (r runsKept) Value(key string) (json.RawMessage, error) {
e, err := r.kv.Get(r.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when
// first used.
type runObjects struct {
ctx context.Context
js jetstream.JetStream
module string
mu sync.Mutex
store jetstream.ObjectStore
}
func (o *runObjects) open() (jetstream.ObjectStore, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.store != nil {
return o.store, nil
}
s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState)))
if err != nil {
return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err)
}
o.store = s
return s, nil
}
func (o *runObjects) Put(name string, data []byte) error {
s, err := o.open()
if err != nil {
return err
}
_, err = s.PutBytes(o.ctx, name, data)
return err
}
func (o *runObjects) Get(name string) ([]byte, error) {
s, err := o.open()
if err != nil {
return nil, err
}
data, err := s.GetBytes(o.ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, pipeline.ErrNoRun
}
return data, err
}
func (o *runObjects) Delete(name string) error {
s, err := o.open()
if err != nil {
return err
}
if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting.
func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) {
return func() (seat.Fill, error) {
st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState))
if err != nil {
return seat.Fill{}, err
}
info, err := st.Info(ctx)
if err != nil {
return seat.Fill{}, err
}
at := now().UTC()
return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil
}
}
// The files the manifest writes each setting of the test runs into, `${config:<key>}`, named by the environment (A-469x).
const (
envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE"
envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE"
envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE"
envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE"
)
// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long.
const maxRunsKept = 168 * time.Hour
// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB,
// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its
// default; a content that is neither is an error naming the file.
func runSettings(getenv func(string) string) (pipeline.RunSettings, error) {
s := pipeline.DefaultRunSettings
if v, err := settingText(getenv(envRunsCap)); err != nil {
return s, err
} else if v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v)
}
s.CapBytes = int64(n) << 20
}
for _, d := range []struct {
env string
to *time.Duration
}{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} {
v, err := settingText(getenv(d.env))
if err != nil {
return s, err
}
if v == "" {
continue
}
t, err := time.ParseDuration(v)
if err != nil || t <= 0 {
return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v)
}
*d.to = t
}
if s.Kept > maxRunsKept {
return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept)
}
return s, nil
}
// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one.
func settingText(path string) (string, error) {
if path == "" {
return "", nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return "", nil
}
if err != nil {
return "", err
}
return strings.TrimSpace(string(raw)), nil
}
// progressSubjects hear every build seat's step-progress (§725), from their contracts.
var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll}
// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p).
func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error {
for _, subject := range progressSubjects {
err := hub.Handle(subject, func(m jetstream.Msg) {
var ev nodebuildgo.StepProgress
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
logf("%s is no step-progress: %v", m.Subject(), err)
_ = m.Term()
return
}
if err := progress(ev.Ask); err != nil {
logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err)
_ = m.Nak()
return
}
_ = m.Ack()
})
if err != nil {
return err
}
}
return nil
}
// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the
// process.
func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error {
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return err
}
go func() {
for {
select {
case <-ctx.Done():
_ = w.Stop()
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) {
continue
}
if err := expired(e.Key()); err != nil {
logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err)
}
}
}
}()
return nil
}
// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725).
func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string,
starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) {
settings, err := runSettings(getenv)
if err != nil {
return nil, nil, err
}
kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run")
if err != nil {
return nil, nil, err
}
runs := testRunsKV{ctx: ctx, js: js, kv: kv}
objects := &runObjects{ctx: ctx, js: js, module: cred.Module}
starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings
starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now)
starter.EndReaders = hearer.EndReaders
hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop
hearer.RunOutputs = objects
if hearer.Ager != nil {
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept
}
logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken)
return kv, objects, nil
}
@@ -0,0 +1,536 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/testbus"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
"mesh-delivery-pipeline/internal/pipeline"
)
// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in
// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a
// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill.
// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes.
func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// test-runs' own object store, made with the state (A-469e).
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
// The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go
// has a holder, node-build-node none.
holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module)
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
t.Fatal(err)
}
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
// integration step optional; web is built by node-build-node.
policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}}
regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy},
{Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}}
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
entries := []catalog.CatalogEntry{
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}},
{Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}},
}
var revision atomic.Int64
revision.Store(1)
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
r := revision.Load()
raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
if in.Seat != "" {
entry.Definition = def
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
}
raw, _ := json.Marshal(map[string]any{"result": result})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
// Every pipeline-ended said, and every drop asked of a build seat.
var mu sync.Mutex
var endings []seat.PipelineEnded
var drops []string
if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) {
var e seat.PipelineEnded
if json.Unmarshal(m.Data, &e) == nil {
mu.Lock()
endings = append(endings, e)
mu.Unlock()
}
}); err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) {
mu.Lock()
drops = append(drops, string(m.Data))
mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": map[string]any{}})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
ended := func(key string) []seat.PipelineEnded {
mu.Lock()
defer mu.Unlock()
var out []seat.PipelineEnded
for _, e := range endings {
if e.Pipeline == key {
out = append(out, e)
}
}
return out
}
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds.
dir := t.TempDir()
files := map[string]string{envRunsCap: "1", envStepBound: "2s"}
env := map[string]string{}
for name, content := range files {
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
env[name] = path
}
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
env[process.BrokerFileEnv] = broker
logs := &syncBuffer{}
start := func() func() {
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
return func() { stop(); <-done }
}
stop := start()
defer func() { stop() }()
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState)
if err != nil {
t.Fatal(err)
}
firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState)
if err != nil {
t.Fatal(err)
}
// The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y).
waitFor(t, ctx, logs, func() bool {
e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders"))
var s views.Status
return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero()
})
takeIn := func(module, commit, fp string, source, rev int64) {
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev,
Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
t.Fatal(err)
}
}
placed := func(key string) seat.Pipeline {
var p seat.Pipeline
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 })
return p
}
read := func(key string) seat.Pipeline {
var p seat.Pipeline
readPipeline(ctx, pipelines, key, &p)
return p
}
step := func(p seat.Pipeline, artifact, name string) seat.StepResult {
for _, s := range p.Steps {
if s.Artifact == artifact && s.Step == name {
return s
}
}
return seat.StepResult{}
}
runOf := func(key string) (pipeline.Run, bool) {
e, err := runsKV.Get(ctx, key)
var r pipeline.Run
return r, err == nil && json.Unmarshal(e.Value(), &r) == nil
}
runKeys := func(candidate string) []string {
ks, _ := runsKV.Keys(ctx)
var out []string
for _, k := range ks {
if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") {
out = append(out, k)
}
}
return out
}
runOfStep := func(key, artifact, name string) (string, pipeline.Run) {
var run string
var r pipeline.Run
waitFor(t, ctx, logs, func() bool {
s := step(read(key), artifact, name)
if len(s.Shared) == 0 {
return false
}
run = s.Shared[0].Run
got, ok := runOf(run)
r = got
return ok && r.Ask != ""
})
return run, r
}
publish := func(subject string, body any) {
raw, _ := json.Marshal(body)
if _, err := js.Publish(ctx, subject, raw); err != nil {
t.Fatal(err)
}
}
taken := func(ask string) {
publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()})
}
endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) {
now := time.Now()
publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1",
Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now})
}
passBuild := func(key, module, commit, fp string) {
ask := step(read(key), "", seat.StepResultStepBuild).Ask
publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
}
call := func(verb string, in, out any) string {
var reply struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
waitFor(t, ctx, logs, func() bool {
m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second)
reply.Error = ""
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
})
if out != nil && reply.Error == "" {
_ = json.Unmarshal(reply.Result, out)
}
return reply.Error
}
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
const p3 = "main.3.app"
takeIn("app", "c1", "src2:aa", 3, 1)
passBuild(p3, "app", "c1", "src2:aa")
unitRun, unit := runOfStep(p3, "app:program", "unit")
integRun, integ := runOfStep(p3, "app:program", "integration")
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
}
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
}
// A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both.
taken(unit.Ask)
waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" })
// Its clock holds the run's ask (A-470n).
waitFor(t, ctx, logs, func() bool {
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`)
})
stop()
stop = start()
// It keeps saying its progress past its bound and is not ended (§725, §720).
// Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises).
boundRev := func() uint64 {
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
if err != nil {
return 0
}
return e.Revision()
}
for at := time.Now(); time.Since(at) < 3*time.Second; {
before := boundRev()
publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()})
waitFor(t, ctx, logs, func() bool { return boundRev() > before })
}
if r, _ := runOf(unitRun); r.Ended != nil {
t.Fatalf("a run saying its progress past its bound ended: %+v", r)
}
// A failing test fails once and is counted once (§726).
endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5})
taken(integ.Ask)
endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2})
waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 })
p := read(p3)
for _, a := range []string{"app:program", "app:tools"} {
if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun {
t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s)
}
}
for _, k := range []string{unitRun, integRun} {
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed {
t.Errorf("the run %s at its end = %+v; want it ended failed", k, r)
}
}
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 {
t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e)
}
// An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the
// pipeline says pipeline-ended again (§727, A-470d).
if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" {
t.Fatalf("accept: %s", e)
}
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" {
t.Fatalf("rerun: %s", e)
}
var again pipeline.Run
waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask })
taken(again.Ask)
endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5})
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" {
t.Fatalf("rerun: %s", e)
}
var integAgain pipeline.Run
waitFor(t, ctx, logs, func() bool {
r, ok := runOf(integRun)
integAgain = r
return ok && r.Ended == nil && r.Ask != integ.Ask
})
taken(integAgain.Ask)
endAsk(p3, integAgain.Ask, "integration", "passed", nil)
waitFor(t, ctx, logs, func() bool {
e := ended(p3)
return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed
})
p = read(p3)
if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun {
t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s)
}
if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted {
t.Errorf("the accepted step = %+v; want it accepted still", s)
}
for _, k := range []string{unitRun, integRun} {
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed {
t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r)
}
}
if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount {
t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount)
}
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
revision.Store(2)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
const p4, p5 = "main.4.app", "main.5.app"
takeIn("app", "c1", "src2:aa", 4, 2)
placed(p4)
run4, r4 := runOfStep(p4, "app:program", "integration")
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
}
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
takeIn("app", "c1", "src2:aa", 5, 2)
placed(p5)
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
t.Fatalf("cancel: %s", e)
}
waitFor(t, ctx, logs, func() bool {
r, _ := runOf(run4)
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
})
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
}
mu.Lock()
dropped := append([]string(nil), drops...)
mu.Unlock()
if len(dropped) != 0 {
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
}
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
taken(r4.Ask)
waitFor(t, ctx, logs, func() bool {
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
})
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
}
waitFor(t, ctx, logs, func() bool {
mu.Lock()
defer mu.Unlock()
return len(drops) == 1 && strings.Contains(drops[0], r4.Ask)
})
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
// asked (§535, §647; A-470g, A-470h).
const web = "main.5.web"
takeIn("web", "w1", "src2:ww", 5, 2)
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
}
// The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked,
// and none is asked (A-470h).
for _, s := range read(web).Steps {
want := seat.StepResultOutcomeNotAsked
switch {
case s.Status == seat.StepResultStatusBypass:
want = seat.StepResultOutcomeBypassed
case s.Seat == "node-build-node":
want = seat.StepResultOutcomeCouldNotRun
}
if s.Outcome != want || s.Ask != "" {
t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want)
}
}
// test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output
// object; test-runs-drop deletes the key and its object (§728).
var runs seat.TestRunsAnswer
if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState {
t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e)
}
objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState)))
if err != nil {
t.Fatal(err)
}
if _, err := objects.GetInfo(ctx, unitRun); err != nil {
t.Fatalf("the unit run's output object: %v", err)
}
var drop seat.TestRunsDropAnswer
if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun {
t.Fatalf("test-runs-drop = %+v (%s)", drop, e)
}
if _, ok := runOf(unitRun); ok {
t.Error("the dropped run's key is kept")
}
if _, err := objects.GetInfo(ctx, unitRun); err == nil {
t.Error("the dropped run's output object is kept")
}
// At the cap a step ends could not run, naming the fill (§725, A-469d).
pad := make([]byte, 600<<10)
for _, k := range []string{"pad.a", "pad.b"} {
if _, err := runsKV.Put(ctx, k, pad); err != nil {
t.Fatal(err)
}
}
revision.Store(3)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
const p6 = "main.6.app"
takeIn("app", "c1", "src2:aa", 6, 3)
waitFor(t, ctx, logs, func() bool {
s := step(read(p6), "app:program", "unit")
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
})
}
func mustJSON(v any) []byte {
raw, _ := json.Marshal(v)
return raw
}
@@ -0,0 +1,58 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
)
// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the
// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time
// past 168h, an error naming the file.
func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
rows := []struct {
name string
files map[string]string
ok bool
check func(t *testing.T, cap int64, kept, step, untaken time.Duration)
}{
{"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour {
t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken)
}
}},
{"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true,
func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute {
t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken)
}
}},
{"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil},
{"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil},
{"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
env := map[string]string{}
for k, v := range r.files {
env[k] = write(r.name+k, v)
}
s, err := runSettings(func(k string) string { return env[k] })
if (err == nil) != r.ok {
t.Fatalf("runSettings = %+v, %v", s, err)
}
if r.check != nil {
r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken)
}
})
}
}
@@ -1,13 +1,10 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -41,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
}
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and
// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i).
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
@@ -51,24 +49,17 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
}
}
// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on.
func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error {
return func(ask string) error {
if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }),
// A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against.
seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }),
}
}
// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts.
func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool),
calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs {
func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs {
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
}
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)}
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
}
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
got = append(got, v)
}
slices.Sort(got)
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
slices.Sort(want)
if !slices.Equal(got, want) {
t.Errorf("the process answers %v, want %v", got, want)
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"time"
@@ -12,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-pipeline/internal/pipeline"
@@ -26,6 +28,70 @@ type runnerViews struct {
// Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own
// findings (A-206).
Findings *findingsWriter
// Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z).
Holders *holderView
}
// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat
// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders`
// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's.
// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances.
type holderView struct {
reader *views.ProcessReader
catalog *pipeline.Views
logf func(string, ...any)
mu sync.Mutex
owner string
pic *views.Picture
}
// refresh follows the claimant's seat-holders view when the claimant is another than the one followed.
func (h *holderView) refresh() {
var halves []pipeline.Half
for _, b := range h.catalog.Branches() {
if half, ok := h.catalog.View(b); ok {
halves = append(halves, half)
}
}
claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat)
h.mu.Lock()
defer h.mu.Unlock()
if !ok || claimant == h.owner {
return
}
if h.owner != "" {
if err := h.reader.Drop(h.owner, seatHoldersState); err != nil {
h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err)
}
h.owner, h.pic = "", nil
}
pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
if err != nil {
h.logf("the seat-holders view of %s is not followed: %v", claimant, err)
return
}
h.owner, h.pic = claimant, pic
h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat)
}
// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699).
const seatHoldersState = "seat-holders"
// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with
// no claimant followed, nothing is judged (A-466y).
func (h *holderView) Holders(seatName string) ([]string, bool) {
h.mu.Lock()
pic := h.pic
h.mu.Unlock()
if pic == nil {
return nil, false
}
select {
case <-pic.Follower().Ready():
default:
return nil, false
}
return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true
}
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
@@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
return nil, err
}
reader.Start(ctx, js, cred, status)
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
}
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
+2
View File
@@ -9,6 +9,8 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
@@ -1,6 +1,7 @@
package pipeline
import (
"errors"
"fmt"
"time"
@@ -53,7 +54,12 @@ type Ager struct {
Get(key string) (seat.KeptResult, bool, error)
Delete(key string) error
}
Now func() time.Time
// Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting
// test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a).
Runs TestRuns
RunOutputs func(name string) error
RunsKept time.Duration
Now func() time.Time
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
Logf func(string, ...any)
}
@@ -69,7 +75,16 @@ func (a *Ager) Age() {
if a == nil || a.Log == nil {
return
}
for _, r := range []Retention{Retention7, Retention30, Retention90} {
retentions := []Retention{Retention7, Retention30, Retention90}
if a.Runs != nil {
// The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a).
kept := a.RunsKept
if kept <= 0 {
kept = DefaultRunSettings.Kept
}
retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...)
}
for _, r := range retentions {
if err := a.ageAt(r); err != nil {
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
}
@@ -103,8 +118,57 @@ func (a *Ager) ageAt(r Retention) error {
return a.Aged.Put(r.Name, moved)
}
// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725).
const RetentionTestRuns = "test-runs"
// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the
// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run.
func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
if e.What != seat.LogEntryWhatEnded {
return
}
p, _, err := a.Pipelines.Get(e.Pipeline)
if err != nil {
return
}
run := ""
for _, s := range p.Steps {
if s.Step == e.Step && s.Artifact == e.Artifact {
run = sharedRun(s)
}
}
if run == "" {
return
}
for {
t, rev, err := a.Runs.Get(run)
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
// ended yet is aged at a later reader's end.
if err != nil || t.Ended == nil {
return
}
if t.Output != "" && a.RunOutputs != nil {
if err := a.RunOutputs(t.Output); err != nil {
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
return
}
}
// Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept.
if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
a.logf("aged: the test run %s is not deleted: %v", run, err)
return
}
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
return
}
}
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
switch r.Name {
case RetentionTestRuns:
a.ageRun(r, e)
case Retention7.Name:
if e.Output != "" && a.Outputs != nil {
if err := a.Outputs(e.Output); err != nil {
@@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int {
}
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
return -1
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -1,20 +1,19 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
)
// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five
// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer.
const AskBound = 5 * time.Minute
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
const DropBound = 5 * time.Minute
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
type TakenSeat struct {
@@ -23,15 +22,15 @@ type TakenSeat struct {
Subject string
}
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken,
// so no bound is armed for their asks.
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken.
var TakenSeats = []TakenSeat{
{Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll},
{Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll},
{Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll},
}
// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound.
// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share
// a run (§724).
func BoundSeat(name string) bool {
for _, s := range TakenSeats {
if s.Seat == name {
@@ -41,63 +40,6 @@ func BoundSeat(name string) bool {
return false
}
// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own
// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it.
type Arm func(ask string) error
// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32,
// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never
// a silent wait. A step that ended or was never asked of a seat takes nothing.
func (h *Hearer) NotTaken(ask string, bound time.Duration) error {
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 {
continue
}
return h.endNotTaken(key, ask, bound)
}
return nil
}
func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
for {
p, rev, err := h.Store.Get(key)
if err != nil {
return fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning {
return nil
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now
s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound)
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
ended := endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
}
h.appendUnlogged(key)
h.Ager.Age()
h.Fills.Check()
return h.said(p, false, ended)
}
}
// IsAskKey says whether a key of the state `bounds` is an ask's bound.
func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") }
// DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop.<build seat>.<ask>` (design 47 G
// A-222, A-77): its five-minute expiry says the drop unanswered.
const DropBoundPrefix = "drop."
@@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
said = nil
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil {
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
t.Fatalf("supersede %d (%v)", n, err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
@@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
// work queues, one per running step.
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
return place(p, reg, entries, now, asker, true)
}
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
// naming the run's ask and the step that asked it (A-469m, A-469q).
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
entry := entries[p.Module]
candidate, named := CandidateOf(p)
share = share && named
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
var steps []seat.StepResult
@@ -118,8 +129,27 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
if share && BoundSeat(bs) {
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
run := TestRunKey(candidate, step, digest)
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
if j, asked := askOfRun[run]; asked {
// Another artifact's step of this pipeline asks the run: this one reads it.
first := &asks[j]
first.Readers = append(first.Readers, reader)
s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
steps = append(steps, s)
continue
}
// A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b).
s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
askOfRun[run] = len(asks)
}
steps = append(steps, s)
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
if a.Package != nil {
ask.Package = a.Package.Name
}
@@ -156,6 +186,10 @@ type Asked struct {
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
// are the ask's (§25, §416).
Package string
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
Run, Level, Digest string
Readers []seat.TestRunReader
}
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
@@ -170,8 +204,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put.
func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error {
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error {
var first error
for _, a := range asks {
accept, ok := acceptStep[a.Seat]
@@ -179,15 +214,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e
accept = to
}
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
if ok {
// A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647).
if arm != nil && BoundSeat(a.Seat) {
if aerr := arm(a.ID); aerr != nil {
err = fmt.Errorf("its bound is not armed: %w", aerr)
ok = false
}
}
}
if ok {
err = asker.AskStep(accept, a.ID, a.Ask)
}
@@ -328,3 +328,207 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
}
}
// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read.
type testRunsMem map[string]struct {
r Run
rev uint64
}
func (m testRunsMem) Create(key string, r Run) error {
if _, ok := m[key]; ok {
return ErrExists
}
m[key] = struct {
r Run
rev uint64
}{r, 1}
return nil
}
func (m testRunsMem) Get(key string) (Run, uint64, error) {
e, ok := m[key]
if !ok {
return Run{}, 0, ErrNoRun
}
e.r.Readers = append([]RunReader(nil), e.r.Readers...)
return e.r, e.rev, nil
}
func (m testRunsMem) Delete(key string, rev uint64) error {
if e, ok := m[key]; ok && e.rev != rev {
return ErrChanged
}
delete(m, key)
return nil
}
func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) {
if m[key].rev != rev {
return 0, ErrChanged
}
m[key] = struct {
r Run
rev uint64
}{r, rev + 1}
return rev + 1, nil
}
// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle
// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second
// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same
// digests asks its own runs.
func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{}
s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks,
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
start := func(source, revision int64) seat.Pipeline {
t.Helper()
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
t.Fatalf("start %d = %v, %v", source, ok, err)
}
p, _, _ := store.Get(Key("main", source, "app"))
return p
}
testAsks := func(from int) []string {
var out []string
for _, a := range asks.asks[from:] {
if b, ok := a.(nodebuildgo.StepAsk); ok {
out = append(out, b.Step.Artifact+" "+b.Step.Step)
}
}
return out
}
p := start(4, 9)
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
}
for _, level := range buildSteps {
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
}
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
}
r, _, err := runs.Get(asker.Shared[0].Run)
if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") {
t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err)
}
}
// A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs.
n := len(asks.asks)
q := start(5, 9)
if got := testAsks(n); len(got) != 0 {
t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got)
}
unit := stepNamed(q, "app:program", "unit")
if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit)
}
if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 {
t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers)
}
// A run ended could not run is never read: the next step of its key asks it again.
key := stepNamed(p, "app:program", "lint").Shared[0].Run
r, rev, _ := runs.Get(key)
ended := time.Unix(9, 0)
r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun
_, _ = runs.Update(key, r, rev)
n = len(asks.asks)
start(6, 9)
if got := strings.Join(testAsks(n), "|"); got != "app:program lint" {
t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got)
}
// A second revision with the same digests asks its own runs, so a failed test runs again.
n = len(asks.asks)
start(7, 10)
if got := len(testAsks(n)); got != 3 {
t.Errorf("a second revision asked %d test steps, want its own three", got)
}
}
// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them):
// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat,
// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h);
// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of
// the view is the one module the catalog halves name.
func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
reg, entries := placeFixture()
for _, c := range []struct {
name string
holders []string
read bool
asked bool
}{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} {
t.Run(c.name, func(t *testing.T) {
store, asks := memStore{}, &stepAsks{}
var said []int
s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Holders: func(string) ([]string, bool) { return c.holders, c.read },
Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := store.Get("main.4.app")
lint := stepNamed(p, "app:program", "lint")
asked := false
for _, id := range asks.ids {
asked = asked || id == lint.Ask
}
if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) {
t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked)
}
if c.asked {
if p.Outcome != seat.PipelineOutcomeRunning || said != nil {
t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said)
}
return
}
if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 {
t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids))
}
if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 {
t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want)
}
ended := 0
for _, st := range p.Steps {
switch {
case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked:
ended++
case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "":
// ended by Place itself: a bypass, or a language no build seat names
default:
t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome)
}
}
if ended < 3 || len(said) != ended {
t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said))
}
})
}
claims := func(seats ...string) map[string]any {
var cs []any
for _, n := range seats {
cs = append(cs, map[string]any{"name": n, "scope": "mesh"})
}
return map[string]any{"module": "x", "version": "0.1", "claims": cs}
}
half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")},
"app": {Module: "app", Manifest: claims()}}}
if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" {
t.Errorf("the claimant = %q, %v", got, ok)
}
other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}}
if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok {
t.Errorf("halves that disagree name %q", got)
}
}
@@ -95,12 +95,12 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
if s.Ended != nil || s.Started == nil {
continue
}
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step,
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1})
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)})
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
@@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time {
}
return *p.Started
}
// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m).
func askOfWait(s seat.StepResult) string { return s.Ask }
// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key
// and ask, as one entry with no machine and no asker.
func sharedOfWait(s seat.StepResult) []seat.Shared {
if len(s.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(s.Shared))
for _, x := range s.Shared {
out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine})
}
return out
}
@@ -7,6 +7,7 @@ import (
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
t.Errorf("show with the fill unread: %v", err)
}
}
// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared
// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step,
// the run it waits on (§731; A-469r, A-469v).
func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit")
started := map[string]seat.StepResult{}
for _, e := range f.said {
started[e.Result.Artifact+" "+e.Result.Step] = e.Result
}
for _, s := range []seat.StepResult{asker, reader} {
got, ok := started[s.Artifact+" unit"]
if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) {
t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok)
}
}
if _, ok := started[" composition"]; !ok {
t.Error("the composition step's step-started is not said")
}
pipes := memKept{}
put(t, pipes, p.Pipeline, f.pipeline(p))
a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands()
if err != nil {
t.Fatal(err)
}
waits := map[string]seat.Wait{}
for _, w := range a.Stands {
waits[w.Artifact+" "+w.Step] = w
}
want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}}
if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] {
t.Errorf("the asking step stands as %+v, want its ask and its run", w)
}
if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] {
t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w)
}
if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 {
t.Errorf("the composition step stands as %+v, want its own ask and no run", w)
}
// Resume asks the steps again and says their step-started again (A-469a).
n := len(f.said)
if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil {
t.Fatal(err)
}
if len(f.said) == n {
t.Error("resume said no step-started")
}
}
@@ -2,12 +2,14 @@ package pipeline
import (
"context"
"encoding/json"
"fmt"
"sort"
"sync"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
@@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) {
}
return false, false
}
// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose
// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none.
func ClaimantOf(halves []Half, seatName string) (string, bool) {
found := ""
for _, h := range halves {
var named []string
for _, e := range h.Entries {
raw, err := json.Marshal(e.Manifest)
if err != nil {
continue
}
m, err := catalogue.Decode(raw)
if err != nil {
continue
}
for _, c := range m.Claims {
if c.Name == seatName {
named = append(named, e.Module)
}
}
}
switch {
case len(named) == 0:
continue
case len(named) > 1, found != "" && found != named[0]:
return "", false
}
found = named[0]
}
return found, found != ""
}
@@ -229,9 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
}
p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision
var asks []Asked
var ended []int
if s.Registration != nil {
if reg, ok := s.Registration(p.Module); ok {
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline is placed with no step", key)
}
@@ -246,9 +248,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
return true, err
}
}
@@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) {
if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil {
t.Error("an entry the members do not change got a pipeline")
}
if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 {
// Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a).
started, steps := 0, 0
for _, x := range said {
started += strings.Count(x, seat.EventPipelineStarted+".")
steps += strings.Count(x, seat.EventStepStarted+".")
}
if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
t.Errorf("placing said %v and asked %d steps", said, len(asks.ids))
}
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
@@ -0,0 +1,488 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs
// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of
// an ask never taken.
type RunSettings struct {
CapBytes int64
Kept time.Duration
StepBound time.Duration
Untaken time.Duration
}
// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours.
var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.<run key>`,
// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.<run key>`,
// written at the run's start.
const (
BoundPrefix = "bound."
UntakenPrefix = "untaken."
)
// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives.
type RunObjects interface {
Put(name string, data []byte) error
// Get reads an object; ErrNoRun when it is gone.
Get(name string) ([]byte, error)
Delete(name string) error
}
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the
// run it bounds (A-470n).
type RunClock interface {
// Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision
// (retried when another write came between); it answers the revision written.
Bound(key, ask string, ttl time.Duration) (uint64, error)
// Bounded reads key: the ask it holds and its revision; ok false when it is not written.
Bounded(key string) (ask string, revision uint64, ok bool, err error)
// Unbound deletes key; a key gone is no fault.
Unbound(key string) error
// UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault.
UnboundAt(key string, revision uint64) error
// Has says whether key is written.
Has(key string) (bool, error)
// Keys are test-runs' keys.
Keys() ([]string, error)
}
// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound.
func ParseRunBound(key string) (run string, untaken, ok bool) {
if run, ok = strings.CutPrefix(key, BoundPrefix); ok {
return run, false, run != ""
}
if run, ok = strings.CutPrefix(key, UntakenPrefix); ok {
return run, true, run != ""
}
return "", false, false
}
// IsRunBound says whether a key of test-runs is a bound key rather than a run.
func IsRunBound(key string) bool {
_, _, ok := ParseRunBound(key)
return ok
}
// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines.
func (h *Hearer) runOfAsk(ask string) (string, bool) {
if h.TestRuns == nil || ask == "" {
return "", false
}
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return "", false
}
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil {
continue
}
if i := stepOfAsk(p, ask); i >= 0 {
run := sharedRun(p.Steps[i])
return run, run != ""
}
}
return "", false
}
// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken
// bound deleted and its clock begun, `bound.<run key>` written with test-step-bound as its time to live. An ask of no test
// run takes nothing.
func (h *Hearer) RunTaken(ask, node string) error {
run, ok := h.runOfAsk(ask)
if !ok {
return nil
}
for {
r, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if r.Ask != ask || r.Ended != nil {
h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
return nil
}
if r.Node != node {
r.Node = node
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("the test run %s is not written: %w", run, err)
}
}
h.runStarted(run, r)
break
}
if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil {
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
}
// The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept.
held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run)
if err != nil || !ok || held != ask {
return err
}
return h.Clock.UnboundAt(UntakenPrefix+run, rev)
}
// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run
// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no
// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept.
func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error {
rev, err := clock.Bound(key, ask, ttl)
if err != nil {
return err
}
r, _, err := runs.Get(run)
if err != nil && !errors.Is(err, ErrNoRun) {
return err
}
if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask {
return clock.UnboundAt(key, rev)
}
return nil
}
// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that
// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing.
func (h *Hearer) RunProgress(ask string) error {
run, ok := h.runOfAsk(ask)
if !ok {
return nil
}
r, _, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) {
h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound)
}
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask
// was taken, and a bound of a run ended already, end nothing.
func (h *Hearer) RunExpired(key string) error {
run, untaken, ok := ParseRunBound(key)
if !ok || h.TestRuns == nil {
return nil
}
r, _, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
// An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing.
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
return nil
}
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again,
// so an expiry heard after the bound was armed again for the run's own ask ends nothing.
if _, _, armed, err := h.Clock.Bounded(key); err != nil {
return fmt.Errorf("the bound %s is not read again: %w", key, err)
} else if armed {
h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key)
return nil
}
s := h.runSettings()
why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound)
if untaken {
why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken)
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err)
}
}
return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
}
// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an
// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It
// says how many it ended.
func (h *Hearer) TakeUp() (int, error) {
if h.TestRuns == nil || h.Clock == nil {
return 0, nil
}
all, err := h.Clock.Keys()
if err != nil {
return 0, fmt.Errorf("the test runs are not listed: %w", err)
}
ended := 0
var first error
for _, run := range all {
if IsRunBound(run) {
continue
}
r, _, err := h.TestRuns.Get(run)
if err != nil || r.Ended != nil {
continue
}
bound := UntakenPrefix + run
if r.Node != "" {
bound = BoundPrefix + run
}
has, err := h.Clock.Has(bound)
if err != nil || has {
if err != nil && first == nil {
first = err
}
continue
}
if err := h.RunExpired(bound); err != nil && first == nil {
first = err
}
ended++
}
return ended, first
}
func (h *Hearer) runSettings() RunSettings {
s := h.Runs
d := DefaultRunSettings
if s.CapBytes <= 0 {
s.CapBytes = d.CapBytes
}
if s.Kept <= 0 {
s.Kept = d.Kept
}
if s.StepBound <= 0 {
s.StepBound = d.StepBound
}
if s.Untaken <= 0 {
s.Untaken = d.Untaken
}
return s
}
// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set,
// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not
// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not
// ended by a judgement of the one before.
func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error {
var r Run
for {
got, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) {
return nil
}
now := h.now()
got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests
if end.Node != "" {
got.Node = end.Node
}
if end.Output != "" {
got.Output = end.Output
}
if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("the test run %s is not written: %w", run, err)
}
r = got
break
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
if err := h.Clock.Unbound(k); err != nil {
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
}
}
}
var first error
for _, reader := range r.Readers {
if skip != nil && reader.Of() == *skip {
continue
}
if err := h.endReader(reader, run, r); err != nil && first == nil {
first = err
}
}
return first
}
// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming
// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d,
// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming
// the new ask, its pipeline reopened (§727).
func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
for {
p, rev, err := h.Store.Get(reader.Pipeline)
if err != nil {
return nil // a pipeline gone has no reader left to end
}
i, moved := -1, false
for j, s := range p.Steps {
if s.Artifact != reader.Artifact || sharedRun(s) != run {
continue
}
if open(s) {
i, moved = j, false
} else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil {
i, moved = j, true
}
}
if i < 0 {
return nil
}
now := h.now()
s := &p.Steps[i]
if moved {
s.Rerun, s.By = true, reader.By
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
}
}
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil
if r.Tests != nil {
t := *r.Tests
s.Tests = &t
}
p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now))
asked, err := askApproval(reader.Pipeline, &p, now, h.Asker)
if err != nil {
return err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
}
h.appendUnlogged(reader.Pipeline)
return h.said(p, asked, ended)
}
}
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
// candidate starts a new run.
const NoReaderLeft = "no reader is left"
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
// nothing.
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
run := sharedRun(s)
if run == "" || h.TestRuns == nil {
return
}
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
for {
r, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return
}
if err != nil {
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
return
}
left := r.Readers[:0:0]
for _, o := range r.Readers {
if o.Of() != reader {
left = append(left, o)
}
}
if len(left) == len(r.Readers) {
return
}
r.Readers = left
// The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks
// its own, never joining a run then ended for having no reader.
last := len(left) == 0 && r.Ended == nil
if last {
now := h.now()
r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft
}
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
return
}
if !last {
return
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
if err := h.Clock.Unbound(k); err != nil {
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
}
}
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
}
}
return
}
}
// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask
// (A-469b).
func open(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
}
// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b),
// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732).
func (h *Hearer) runStarted(run string, r Run) {
for _, reader := range r.Readers {
for {
p, rev, err := h.Store.Get(reader.Pipeline)
if err != nil {
break
}
var moved []int
for j, s := range p.Steps {
if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode {
p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, ""
moved = append(moved, j)
}
}
if len(moved) == 0 {
break
}
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err)
break
}
for _, j := range moved {
if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil {
h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err)
}
}
break
}
}
}
@@ -4,8 +4,10 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
@@ -73,8 +75,12 @@ type Starter struct {
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
// none was heard, which asks the step with none.
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
Arm Arm
// Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's
// first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless.
Holders func(seat string) ([]string, bool)
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
Ended func(p seat.Pipeline, steps []int) error
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
Fills *Fills
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
@@ -83,6 +89,116 @@ type Starter struct {
Store Updater
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
Supersede func(newer seat.Pipeline)
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
// (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the
// settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends
// the steps of a run read ended or never asked.
TestRuns TestRuns
Clock RunClock
Runs RunSettings
RunsFill func() (seat.Fill, error)
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
}
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat;
// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless.
func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) {
if s.Holders == nil {
return asks, nil
}
var none []string
for _, st := range p.Steps {
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
continue
}
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
none = append(none, st.Seat)
}
}
if len(none) == 0 {
return asks, nil
}
sort.Strings(none)
whys := make([]string, 0, len(none))
for _, n := range none {
whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n))
}
var ended []int
for i := range p.Steps {
st := &p.Steps[i]
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
continue
}
st.Ended, st.Ask, st.Shared = &now, "", nil
if slices.Contains(none, st.Seat) {
st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat)
} else {
st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ")
}
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
}
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now
p.EndCount++
return nil, ended
}
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
// pipeline's end, through Ended; with none given it is said in the journal alone.
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why)
if s.Ended == nil {
return nil
}
return s.Ended(p, steps)
}
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
// the kept results and toolchain are carried and each ask is put.
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
placed := asks
asks = s.share(key, asks)
var err error
if len(asks) > 0 {
s.carry(key, asks)
err = askAll(s.Steps, key, asks, s.recovered(key, commit))
}
s.sayStarted(key, placed)
return err
}
// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a):
// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times
// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729).
func (s *Starter) sayStarted(key string, asks []Asked) {
if s.Store == nil || s.Say == nil || len(asks) == 0 {
return
}
ids, runs := map[string]bool{}, map[string]bool{}
for _, a := range asks {
ids[a.ID] = true
if a.Run != "" {
runs[a.Run] = true
}
}
p, _, err := s.Store.Get(key)
if err != nil {
s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err)
return
}
at := s.now()
for _, st := range p.Steps {
if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
continue
}
subject, ev := stepEvent(seat.EventStepStarted, p, st, at)
if err := s.Say(subject, ev); err != nil {
s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err)
}
}
}
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
@@ -115,11 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) {
}
p := newPipeline(b, key, now)
var asks []Asked
var ended []int
if b.unheld {
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
} else if s.Registration != nil {
if reg, ok := s.Registration(b.Module); ok {
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
}
@@ -143,9 +261,11 @@ func (s *Starter) Start(b Begin) (bool, error) {
if s.Supersede != nil {
s.Supersede(p)
}
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
if err := s.askSteps(key, b.Commit, asks); err != nil {
return true, err
}
}
@@ -187,24 +307,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if !ok {
continue
}
running := map[string]bool{}
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
running, own := map[string]bool{}, map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
if open(st) && st.Ask != "" {
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
_, shared := Place(p, reg, half.Entries, now, s.Name)
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
var due []Asked
for _, a := range placed {
if running[a.ID] {
for _, a := range shared {
if running[a.ID] && !own[a.ID] {
due = append(due, a)
}
}
for _, a := range unshared {
if running[a.ID] && own[a.ID] {
a.Run = ""
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
s.carry(key, due)
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
first = err
}
asked += len(due)
@@ -284,6 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
}
var asks []Asked
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
asks, ended := s.holderless(&p, asks, now)
if _, err := store.Update(key, p, rev); err != nil {
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
continue
@@ -299,9 +428,14 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
s.logf("pipeline-started of %s is not said: %v", key, err)
}
}
if len(ended) > 0 {
if err := s.endedAtPlacing(p, ended); err != nil {
note(err)
}
continue
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
note(err)
}
}
@@ -26,6 +26,9 @@ type StepSeat struct {
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
}
// StepSeats are the three build seats, from their contracts.
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
// A-469t): the same form, under the same name.
func wantsShared(raw []byte) []seat.Shared {
var w wants.StepEnded
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(w.Shared))
for _, x := range w.Shared {
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
if x.AskedBy != nil {
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
}
out = append(out, e)
}
return out
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
@@ -118,6 +139,26 @@ type Hearer struct {
Fills *Fills
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
Widens func(seat.Pipeline) []seat.Widening
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
TestRuns TestRuns
Clock RunClock
Runs RunSettings
DropRun Drop
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
RunOutputs RunObjects
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
// of a run that was never asked.
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
var first error
for _, reader := range readers {
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
first = err
}
}
return first
}
func (h *Hearer) now() time.Time {
@@ -132,6 +173,13 @@ func (h *Hearer) now() time.Time {
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
return h.OnStepEndedShared(bs, subject, ev, nil)
}
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
@@ -153,7 +201,15 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
case !open(p.Steps[i]):
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
return p, h.endRunOf(p, p.Steps[i], ev)
}
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
}
return p, nil
case !endOutcome(ev.Outcome):
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
@@ -161,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if len(shared) > 0 {
s.Shared = shared
}
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
@@ -177,6 +236,8 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
}
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
p.Unlogged = append(p.Unlogged, unlogged)
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
readers := h.sameRunReaders(&p, i, ev, now)
from := now
if ev.Ended != nil {
from = ev.Ended.UTC()
@@ -191,6 +252,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
// finding's event the bus is slow to take) delays it past its bound (A-470o).
runErr := h.endRunOf(p, *s, ev)
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
@@ -209,14 +273,94 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
for _, j := range readers {
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
}
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
h.appendUnlogged(ev.Pipeline)
h.Ager.Age()
h.Fills.Check()
return p, h.said(p, asked, ended)
if err := h.said(p, asked, ended); err != nil {
return p, err
}
return p, runErr
}
}
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
// answers their indexes.
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
asker := p.Steps[i]
run := sharedRun(asker)
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
return nil
}
marks := map[string]RunReader{}
if r, _, err := h.TestRuns.Get(run); err == nil {
for _, o := range r.Readers {
if o.Pipeline == p.Pipeline {
marks[o.Artifact] = o
}
}
}
var out []int
for j := range p.Steps {
s := &p.Steps[j]
if j == i || sharedRun(*s) != run {
continue
}
switch m := marks[s.Artifact]; {
case open(*s) && s.Ask == "":
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
s.Rerun, s.By = true, m.By
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
default:
continue
}
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
if asker.Tests != nil {
t := *asker.Tests
s.Tests = &t
}
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
out = append(out, j)
}
return out
}
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
// A step that asked no run ends nothing more.
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
run := sharedRun(s)
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
return nil
}
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
if ev.Tests != nil {
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if h.RunOutputs != nil {
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
}
raw, err := json.Marshal(out)
if err == nil {
err = h.RunOutputs.Put(run, raw)
}
if err != nil {
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
}
end.Output = run
}
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
@@ -300,8 +444,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
_, ev := stepEvent(event, p, *step, at)
return h.Say(subject, ev)
case seat.EventPipelineEnded:
e := pipelineEnded(p)
if h.Widens != nil {
@@ -314,17 +458,46 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
}
}
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with
// its shared when it reads a run (§729; A-468e, A-469q).
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
last := p.Commit
if p.Group != "" {
last = p.Group
}
if event == seat.EventStepStarted {
// A step-started names the run and its ask, never the step that asked it (A-469q).
var shared []seat.Shared
for _, x := range step.Shared {
x.AskedBy = nil
shared = append(shared, x)
}
step.Shared = shared
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step}
}
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
const (
budgetUnit = 500
budgetIntegration = 50
)
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
// steps read it (§726, A-466i); nil when no step counted any.
func testCounts(steps []seat.StepResult) *seat.TestCounts {
var t seat.TestCounts
counted := false
runs := map[string]bool{}
for _, s := range steps {
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
if runs[run] {
continue
}
runs[run] = true
}
if s.Tests != nil {
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
}
@@ -380,7 +553,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -401,3 +574,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
}
return true
}
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
var first error
for _, i := range steps {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
}
}
h.appendUnlogged(p.Pipeline)
if len(steps) > 0 {
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
}
if err := h.said(p, false, true); err != nil && first == nil {
first = err
}
return first
}
@@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) {
}
}
// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline
// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing.
func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) {
h, store, said, _, _, _ := fixture()
e := store[pkey]
e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder
e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked
store[pkey] = e
if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
t.Fatalf("an ask no step names ended something: %v", err)
}
if err := h.NotTaken("a1", AskBound); err != nil {
t.Fatal(err)
}
p := store[pkey].p
if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) ||
p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) {
t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat)
}
// Its end taken once: the bound's expiry heard again changes nothing.
n := len(*said)
if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n {
t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err)
}
}
// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is
// not armed is not put (§535, §647).
func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) {
var order []string
asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil })
arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil })
asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}}
if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" {
t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err)
}
order = nil
failing := Arm(func(string) error { return errors.New("bucket gone") })
if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 {
t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err)
}
}
type askerFunc func(accept, id string, body any) error
func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) }
@@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
var to []string
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
}
}
@@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error {
break
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node {
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
break
}
p.Steps[i].Node = node
@@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error {
// whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started
// (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended
// `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through
// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves
// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
// request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended.
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) {
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) {
if newer.Run != "" || newer.Source <= 0 {
return 0, nil
}
@@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri
if key == newer.Pipeline {
continue
}
done, err := h.supersedeOne(key, newer, why, drop, unbound)
done, err := h.supersedeOne(key, newer, why, drop)
if err != nil && first == nil {
first = err
}
@@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string {
return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module)
}
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) {
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) {
for {
p, rev, err := h.Store.Get(key)
if err != nil || !superseded(p, newer) {
@@ -110,10 +110,11 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
var changed []int
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" && BoundSeat(s.Seat) {
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
drops = append(drops, *s)
}
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
@@ -136,17 +137,15 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
if len(changed) > 0 {
h.appendUnlogged(key)
}
for _, i := range changed {
h.leaveRun(key, p.Steps[i], why)
}
for _, s := range drops {
if drop != nil {
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err)
}
}
if unbound != nil {
if err := unbound(s.Ask); err != nil {
h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err)
}
}
}
if err := h.said(p, false, true); err != nil && first == nil {
first = err
@@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) {
}
type dropped struct{ seat, node, ask string }
var drops []dropped
var unbound []string
drop := func(s, node, ask, why string) error {
if !strings.Contains(why, "c2") {
t.Errorf("the drop of %s names no newer commit: %q", ask, why)
@@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) {
drops = append(drops, dropped{s, node, ask})
return nil
}
unbind := func(ask string) error { unbound = append(unbound, ask); return nil }
newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline {
return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub,
Outcome: seat.PipelineOutcomeRunning}
@@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) {
{"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)},
{"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)},
} {
if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil {
if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil {
t.Errorf("%s superseded %d (%v)", c.name, n, err)
}
}
*said = nil
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil {
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil {
t.Fatalf("a changed fingerprint superseded %d (%v)", n, err)
}
p, _, _ := store.Get(pkey)
@@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) {
}
}
want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}}
if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) {
t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want)
if !slices.Equal(drops, want) {
t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want)
}
if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" {
t.Errorf("said %v, want four step-ended then pipeline-ended", *said)
}
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 {
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 {
t.Errorf("a superseded pipeline was superseded again: %d", n)
}
@@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) {
pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa",
Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}}
h.Store = pub
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 {
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 {
t.Errorf("a publish of the same version superseded %d", n)
}
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil {
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil {
t.Fatalf("a newer publish superseded %d (%v)", n, err)
}
if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") {
@@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) {
if _, _, ok := ParseDropBound("step-ab"); ok {
t.Error("an ask's bound was read as a drop's")
}
if IsAskKey(DropBoundKey("s", "step-ab")) {
t.Error("a drop's bound was taken for an ask's")
}
err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why")
if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 {
t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered)
@@ -0,0 +1,374 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested
// packages, written create-only by the step that asks the run and changed by compare-and-set.
type TestRuns interface {
// Create writes key when it is absent; ErrExists when it is written already.
Create(key string, r Run) error
// Get reads key and its revision; ErrNoRun when it is absent.
Get(key string) (Run, uint64, error)
// Update writes key at the revision read; ErrChanged when another write came between.
Update(key string, r Run, revision uint64) (uint64, error)
// Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No
// write of a run puts a value judged from an earlier read over a later one (the kind of the stale write).
Delete(key string, revision uint64) error
}
// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by
// whom (A-470d), so the marks are in the key and a restart takes them up.
type Run struct {
seat.TestRun
Readers []RunReader `json:"readers,omitempty"`
}
// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun
// whose new run moves it (A-470d).
type RunReader struct {
Pipeline string `json:"pipeline"`
Artifact string `json:"artifact"`
Rerun bool `json:"rerun,omitempty"`
By string `json:"by,omitempty"`
}
// Of is the reader as the seat names it.
func (r RunReader) Of() seat.TestRunReader {
return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact}
}
// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact.
func (r Run) Answer() seat.TestRun {
t := r.TestRun
t.Readers = nil
for _, o := range r.Readers {
t.Readers = append(t.Readers, o.Of())
}
return t
}
// ErrNoRun is a Get of a test-runs key that is not written.
var ErrNoRun = errors.New("no test run under the key")
// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-<id>` for a run on request, else `<branch>.<revision>`
// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A
// pipeline naming neither has no candidate, and its steps share no run.
func CandidateOf(p seat.Pipeline) (string, bool) {
var key string
var err error
if p.Run != "" {
key, err = nec.RunKey(p.Run)
} else {
key, err = nec.RevisionKey(p.Branch, p.Revision)
}
return key, err == nil
}
// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint,
// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner
// holds, so two artifacts with one fetch meet and two with different fetches do not.
func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string {
paths := append([]string(nil), fetch.Paths...)
sort.Strings(paths)
h := sha256.New()
for _, part := range []string{fingerprint, level, status, fetch.Dir} {
h.Write([]byte(part + "\n"))
}
h.Write([]byte(strings.Join(paths, "\n")))
return hex.EncodeToString(h.Sum(nil))
}
// TestRunKey is a test run's key in test-runs (§725): `<candidate key>.<level>.<digest>`, the level and the digest each
// escaped as a key's name, the candidate already in its key form.
func TestRunKey(candidate, level, digest string) string {
return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest)
}
// sharedRun is the one run a step's Shared names, empty when it names none.
func sharedRun(s seat.StepResult) string {
if len(s.Shared) == 0 {
return ""
}
return s.Shared[0].Run
}
// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask
// of its own (A-469m).
func readsRun(s seat.StepResult, run string) bool {
return run != "" && sharedRun(s) == run && s.Ask == ""
}
// addReaders adds each reader not in r's readers yet, in order.
func addReaders(r *Run, readers []seat.TestRunReader) {
for _, n := range readers {
have := false
for _, o := range r.Readers {
have = have || o.Of() == n
}
if !have {
r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact})
}
}
}
// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is.
func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
if store == nil || ask == "" {
return nil, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
continue
}
if i := stepOfAsk(p, ask); i >= 0 {
return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil
}
}
return nil, nil
}
// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run;
// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their
// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read:
// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again,
// kept once by its id. It answers the asks still to put.
func (s *Starter) share(key string, asks []Asked) []Asked {
if s.TestRuns == nil {
return asks
}
var out []Asked
joined := map[string]Run{}
for _, a := range asks {
if a.Run == "" {
out = append(out, a)
continue
}
run, ok, err := s.claimRun(a)
var full *fullErr
switch {
case errors.As(err, &full):
// At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d).
s.logf("the test run %s of %s is not asked: %v", a.Run, key, err)
if s.EndReaders != nil {
if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil {
s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err)
}
}
case err != nil:
// A run that cannot be written or read is asked as the step's own, said: slower, never lost.
s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err)
out = append(out, a)
case ok:
out = append(out, a)
default:
joined[a.Run] = run
}
}
if len(joined) > 0 {
s.readOthers(key, joined)
// A run that ended within the candidate is read at once (§724): its readers end with its result.
for run, r := range joined {
if r.Ended == nil || s.EndReaders == nil {
continue
}
var readers []seat.TestRunReader
for _, a := range asks {
if a.Run == run {
readers = a.Readers
}
}
if err := s.EndReaders(run, readers, r.TestRun); err != nil {
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
}
}
}
return out
}
// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill.
type fullErr struct{ fill seat.Fill }
func (e *fullErr) Error() string {
return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap)
}
// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i),
// never the manifest's ceiling.
func (s *Starter) TestRunsFill() (seat.Fill, error) {
if s.RunsFill == nil {
return seat.Fill{}, errors.New("the runner reads no fill of test-runs")
}
f, err := s.RunsFill()
if err != nil {
return seat.Fill{}, err
}
f.Cap = s.Runs.CapBytes
if f.Cap <= 0 {
f.Cap = DefaultRunSettings.CapBytes
}
return f, nil
}
// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said).
func (s *Starter) atCap() error {
if s.RunsFill == nil {
return nil
}
f, err := s.TestRunsFill()
if err != nil {
s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err)
return nil
}
if f.Bytes >= f.Cap {
return &fullErr{fill: f}
}
return nil
}
// started writes a new run and its untaken bound (§725, A-470).
func (s *Starter) started(a Asked, write func() error) error {
if err := s.atCap(); err != nil {
return err
}
if err := write(); err != nil {
return err
}
if s.Clock != nil {
untaken := s.Runs.Untaken
if untaken <= 0 {
untaken = DefaultRunSettings.Untaken
}
if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil {
s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err)
}
}
return nil
}
// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads.
func (s *Starter) claimRun(a Asked) (Run, bool, error) {
now := s.now()
value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}}
addReaders(&value, a.Readers)
for {
err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) })
if err == nil {
return value, true, nil
}
var full *fullErr
if errors.As(err, &full) {
return Run{}, false, err
}
if !errors.Is(err, ErrExists) {
return Run{}, false, err
}
r, rev, err := s.TestRuns.Get(a.Run)
if errors.Is(err, ErrNoRun) {
continue // deleted meanwhile: written again
}
if err != nil {
return Run{}, false, err
}
if r.Ask == a.ID {
return r, true, nil
}
if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun {
err := s.started(a, func() error {
_, err := s.TestRuns.Update(a.Run, value, rev)
return err
})
if errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return Run{}, false, err
}
return value, true, nil
}
addReaders(&r, a.Readers)
if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return Run{}, false, err
}
return r, false, nil
}
}
// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker
// in Shared (A-469m, A-469q).
func (s *Starter) readOthers(key string, joined map[string]Run) {
if s.Store == nil {
return
}
by := map[string]*seat.SharedAskedBy{}
for run, r := range joined {
asker, err := askerOf(s.Store, r.Ask)
if err != nil {
s.logf("the step that asked the test run %s is not read: %v", run, err)
}
by[run] = asker
}
for {
p, rev, err := s.Store.Get(key)
if err != nil {
s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err)
return
}
for i := range p.Steps {
st := &p.Steps[i]
r, ok := joined[sharedRun(*st)]
if !ok || !open(*st) {
continue
}
if r.Node != "" {
st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b)
}
st.Ask = ""
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
}
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err)
}
return
}
}
// joinRun adds reader to run's readers by compare-and-set.
func (s *Starter) joinRun(run string, reader seat.TestRunReader) {
if s.TestRuns == nil {
return
}
for {
r, rev, err := s.TestRuns.Get(run)
if err != nil {
s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err)
return
}
addReaders(&r, []seat.TestRunReader{reader})
if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err)
}
return
}
}
@@ -0,0 +1,176 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the
// passing ones it kept and the carried ones it skipped, and the build seat's output.
type RunOutput struct {
Packages []seat.TestRunPackage `json:"packages"`
Output string `json:"output,omitempty"`
}
// Gone is what a run whose output object is gone answers in place of its packages (§725, §733).
const Gone = "the run's output is gone"
// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object
// store: each run with its packages read from its output object, and the state's fill.
type RunsRead struct {
Runs Kept
// Object reads an object of test-runs-objects; ErrNoRun when it is gone.
Object func(name string) ([]byte, error)
Fill func() (seat.Fill, error)
}
// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one.
func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) {
all, err := r.Runs.Keys()
if err != nil {
return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err)
}
sort.Strings(all)
a := seat.TestRunsAnswer{Runs: []seat.TestRun{}}
for _, key := range all {
if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
continue
}
raw, err := r.Runs.Value(key)
if err != nil || raw == nil {
continue
}
var run Run
if err := json.Unmarshal(raw, &run); err != nil {
continue
}
if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) {
continue
}
t := run.Answer()
t.Key = key
r.packages(&t)
a.Runs = append(a.Runs, t)
}
if r.Fill != nil {
f, err := r.Fill()
if err != nil {
return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err)
}
a.Fill = f
}
return a, nil
}
// packages names t's packages from its output object, or says it gone (§725, A-466v).
func (r RunsRead) packages(t *seat.TestRun) {
if t.Output == "" || r.Object == nil {
return
}
data, err := r.Object(t.Output)
if errors.Is(err, ErrNoRun) {
t.Gone = Gone
return
}
var out RunOutput
if err == nil {
err = json.Unmarshal(data, &out)
}
if err != nil {
t.Gone = "the run's output is not read: " + err.Error()
return
}
t.Packages = out.Packages
}
// readsModule says whether a reader of run is a step of module, its artifact `<module>:<name>`.
func readsModule(run Run, module string) bool {
for _, o := range run.Readers {
if strings.HasPrefix(o.Artifact, module+":") {
return true
}
}
return false
}
// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the
// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output
// object and its key deleted. It answers the run deleted and the readers it ended.
func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) {
h := v.Hearer
if h.TestRuns == nil {
return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs")
}
r, _, err := h.TestRuns.Get(in.Key)
if errors.Is(err, ErrNoRun) {
return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key)
}
if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err)
}
if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil {
return seat.TestRunsDropAnswer{}, err
}
why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why)
var ended []seat.TestRunReader
if r.Ended == nil {
if h.DropRun != nil && r.Ask != "" {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err)
}
}
if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
return seat.TestRunsDropAnswer{}, err
}
for _, o := range r.Readers {
ended = append(ended, o.Of())
}
}
if h.RunOutputs != nil {
// The run's object is named by its key (endRunOf); one the run named otherwise is deleted too.
for _, name := range []string{in.Key, r.Output} {
if name == "" {
continue
}
if err := h.RunOutputs.Delete(name); err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err)
}
}
}
// The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it.
for {
now, rev, err := h.TestRuns.Get(in.Key)
if errors.Is(err, ErrNoRun) {
break
}
if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err)
}
if now.Ask != r.Ask {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask)
}
if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
}
break
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
_ = h.Clock.Unbound(k)
}
}
gone := r.Answer()
gone.Key = in.Key
now := h.now()
if gone.Ended == nil {
gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why
}
return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil
}
@@ -0,0 +1,672 @@
package pipeline
import (
"fmt"
"strings"
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
)
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds
// and its revision.
type clockMem struct {
runs testRunsMem
bound map[string]time.Duration
asks map[string]string
revs map[string]uint64
rev uint64
// before is called once at the next Bound, a write that comes between its caller's read and its write.
before func()
}
func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) {
if b := c.before; b != nil {
c.before = nil
b()
}
if c.asks == nil {
c.asks, c.revs = map[string]string{}, map[string]uint64{}
}
c.rev++
c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev
return c.rev, nil
}
func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
_, ok := c.bound[key]
return c.asks[key], c.revs[key], ok, nil
}
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
func (c *clockMem) UnboundAt(key string, rev uint64) error {
if c.revs[key] == rev {
delete(c.bound, key)
}
return nil
}
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
func (c *clockMem) Keys() ([]string, error) {
var out []string
for k := range c.runs {
out = append(out, k)
}
for k := range c.bound {
out = append(out, k)
}
return out, nil
}
type objectsMem map[string][]byte
func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil }
func (o objectsMem) Get(name string) ([]byte, error) {
if d, ok := o[name]; ok {
return d, nil
}
return nil, ErrNoRun
}
func (o objectsMem) Delete(name string) error { delete(o, name); return nil }
// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing
// one directory.
type runFixture struct {
store memStore
runs testRunsMem
clock *clockMem
objects objectsMem
asks *stepAsks
drops []string
said []seat.StepEvent
logged []string
fill int64
s *Starter
h *Hearer
}
func newRunFixture() *runFixture {
f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}}
f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}}
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
say := func(_ string, body any) error {
if e, ok := body.(seat.StepEvent); ok {
f.said = append(f.said, e)
}
return nil
}
f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects,
Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) },
DropRun: func(seatName, node, ask, why string) error {
f.drops = append(f.drops, seatName+" "+node+" "+ask)
return nil
}}
f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks,
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders,
RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf}
return f
}
func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
t.Helper()
entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := f.store.Get(Key("main", source, "app"))
return p
}
func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline {
q, _, _ := f.store.Get(p.Pipeline)
return q
}
// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken
// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and
// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run
// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended
// run is aged with its output object in the step log's order.
func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
t.Run("clock", func(t *testing.T) {
f := newRunFixture()
f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
f.s.Runs = f.h.Runs
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 {
t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound)
}
// A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again.
if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode {
t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome)
}
n := len(f.said)
if err := f.h.RunTaken(unit.Ask, "ace"); err != nil {
t.Fatal(err)
}
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome)
}
}
if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n)
}
if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound)
}
if _, ok := f.clock.bound[UntakenPrefix+run]; ok {
t.Error("the untaken bound of a taken run is kept")
}
// An untaken bound of a taken run ends nothing; progress begins the clock again.
if err := f.h.RunExpired(UntakenPrefix + run); err != nil {
t.Fatal(err)
}
delete(f.clock.bound, BoundPrefix+run)
if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound)
}
if r, _, _ := f.runs.Get(run); r.Ended != nil {
t.Fatal("a run is ended by its untaken bound after it was taken")
}
// An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n).
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask {
t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run])
}
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
t.Fatal(err)
}
r, _, _ := f.runs.Get(run)
if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") {
t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r)
}
if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask {
t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops)
}
q := f.pipeline(p)
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said {
t.Errorf("the %s unit step = %+v, want could not run with the run", art, s)
}
}
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil {
t.Fatal(err)
}
if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun {
t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome)
}
if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) {
t.Errorf("the ignored step-ended is not journaled: %q", f.logged)
}
// An untaken run whose bound passed while the runner was away ends at the start.
lint := stepNamed(p, "app:program", "lint").Shared[0].Run
delete(f.clock.bound, UntakenPrefix+lint)
if n, err := f.h.TakeUp(); n != 1 || err != nil {
t.Fatalf("take-up ended %d, %v; want the lint run", n, err)
}
if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") {
t.Errorf("the lint run taken up = %+v", r)
}
})
t.Run("cap", func(t *testing.T) {
f := newRunFixture()
f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20
p := f.start(t, 4, 9)
for _, a := range f.asks.ids {
if a == stepNamed(p, "app:program", "unit").Ask {
t.Error("a unit run is asked at the cap")
}
}
q := f.pipeline(p)
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") {
t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s)
}
}
})
t.Run("age", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil)
f.objects["out-1"] = []byte("ok")
now := agedNow
a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store},
Aged: agedState{}, Now: func() time.Time { return now },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
if from > 1 {
return nil, nil
}
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil
}}
a.Age()
if _, _, err := f.runs.Get(run); err != nil {
t.Fatal("a run is aged before test-runs-kept passed")
}
now = now.Add(2 * 24 * time.Hour)
a.Age()
if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 {
t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects)
}
})
}
// agedDeleter is a memStore with the delete the ager makes.
type agedDeleter struct{ memStore }
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
f := newRunFixture()
a := f.start(t, 4, 9)
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
unit := stepNamed(a, "app:program", "unit")
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
t.Fatal(err)
}
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
for _, art := range []string{"app:program", "app:tools"} {
s := stepNamed(p, art, "unit")
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
}
}
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
}
}
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) {
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
}
saidReaders := 0
for _, e := range f.said {
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
saidReaders++
}
}
if saidReaders != 4 {
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
}
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
lint := stepNamed(a, "app:program", "lint")
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
}
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
}
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
c := newRunFixture()
p := c.start(t, 4, 9)
comp := stepNamed(p, "", "composition")
bs := stepSeatOf(t, "mesh-wants")
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
ev, err := bs.Decode(raw)
if err != nil {
t.Fatal(err)
}
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
t.Fatal(err)
}
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
}
}
// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed
// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's
// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the
// pipeline says pipeline-ended again; a step a person accepted does not move.
func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) {
// takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader
// with rerun false, its step passed already, says no step-ended again (A-470l).
for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} {
accepted := c.accepted
f := newRunFixture()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
p := f.start(t, 4, 9)
e := f.store[p.Pipeline]
e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build
f.store[p.Pipeline] = e
unit := stepNamed(p, "app:program", "unit")
end := func(ask string, outcome string, rerun bool) {
t.Helper()
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome,
Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil {
t.Fatal(err)
}
}
end(unit.Ask, "failed", false)
if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" {
t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome)
}
if accepted {
e := f.store[p.Pipeline]
i := stepIndex(e.p, "unit", "app:program")
e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"}
f.store[p.Pipeline] = e
}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
return Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
}, Acts: &actLog{}}
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"})
if err != nil {
t.Fatal(err)
}
again := a.Result.Ask
if c.takeUp {
run := unit.Shared[0].Run
r, rev, _ := f.runs.Get(run)
r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already
if _, err := f.runs.Update(run, r, rev); err != nil {
t.Fatal(err)
}
e := f.store[p.Pipeline]
e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed
f.store[p.Pipeline] = e
for k := range f.clock.bound {
delete(f.clock.bound, k)
}
said := len(f.said)
if n, err := f.h.TakeUp(); err != nil || n != 1 {
t.Fatalf("take-up = %d, %v", n, err)
}
for _, ev := range f.said[said:] {
if ev.Result.Artifact == "app:program" {
t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev)
}
}
q := f.pipeline(p)
if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun {
t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run",
stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome)
}
continue
}
r, _, _ := f.runs.Get(unit.Shared[0].Run)
if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") {
t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again)
}
end(again, "passed", true)
q := f.pipeline(p)
prog := stepNamed(q, "app:program", "unit")
if accepted {
if prog.Outcome != seat.StepResultOutcomeAccepted {
t.Errorf("an accepted step moved to %s", prog.Outcome)
}
continue
}
if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again {
t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog)
}
if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 {
t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount)
}
}
}
// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its
// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the
// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act
// recorded it does nothing.
func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint")
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed",
Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil {
t.Fatal(err)
}
pipes := memKept{}
for k, e := range f.runs {
put(t, pipes, k, e.r)
}
// The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i).
f.s.Runs.CapBytes, f.fill = 1<<20, 9
bucket := f.s.RunsFill
f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err }
read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill}
a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"})
if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) ||
a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 {
t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err)
}
delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages
if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 {
t.Errorf("a run whose output is gone = %+v", a.Runs)
}
acts := &actLog{}
v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts}
if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil {
t.Fatal("a drop with no hand act recorded was done")
}
f.objects[lint.Shared[0].Run] = []byte("{}")
d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"})
if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 {
t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops)
}
if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil {
t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run])
}
if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") {
t.Errorf("a reader of the dropped run = %+v", s)
}
}
// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called
// once, just before that write.
type racingRuns struct {
testRunsMem
beforeUpdate, beforeDelete func()
}
func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) {
if b := r.beforeUpdate; b != nil {
r.beforeUpdate = nil
b()
}
return r.testRunsMem.Update(key, v, rev)
}
func (r *racingRuns) Delete(key string, rev uint64) error {
if b := r.beforeDelete; b != nil {
r.beforeDelete = nil
b()
}
return r.testRunsMem.Delete(key, rev)
}
// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design
// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the
// latest read, and a judgement of a run ends only the run under the ask it judged.
func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
asked := func(f *runFixture, run string) {
t.Helper()
r, rev, _ := f.runs.Get(run)
r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"}
if _, err := f.runs.Update(run, r, rev); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
race func(t *testing.T, f *runFixture, p seat.Pipeline, run string)
}{
{"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
_ = f.h.RunExpired(BoundPrefix + run)
}},
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.DropRun = func(string, string, string, string) error {
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
return nil
}
for _, a := range []string{"app:program", "app:tools"} {
f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled")
}
}},
{"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil)
runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
if from > 1 {
return nil, nil
}
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil
}}
a.Age()
}},
{"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil {
t.Error("a drop whose key holds a run asked since answers no error")
}
}},
{"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) }
_ = f.h.RunProgress(unit.Ask)
if _, ok := f.clock.bound[BoundPrefix+run]; ok {
t.Error("a clock written from a read before the run's end is kept")
}
asked(f, run) // the key's run is asked again, so the check below holds
}},
{"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
asked(f, run)
_ = f.h.RunExpired(BoundPrefix + run)
}},
} {
t.Run(c.name, func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
run := stepNamed(p, "app:program", "unit").Shared[0].Run
c.race(t, f, p, run)
if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil {
t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err)
}
})
}
// A late signal of a replaced run arms no bound and is said in the journal (A-470n).
t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
asked(f, run)
delete(f.clock.bound, UntakenPrefix+run)
_ = f.h.RunTaken(unit.Ask, "ace")
_ = f.h.RunProgress(unit.Ask)
late := 0
for _, l := range f.logged {
if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") {
late++
}
}
if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 {
t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged)
}
})
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
f := newRunFixture()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
endedFirst := false
f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) {
r, _, _ := f.runs.Get(run)
endedFirst = r.Ended != nil
return "alice", nil
}}
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
t.Fatal(err)
}
if !endedFirst {
t.Error("the step's finding was written before its run's end")
}
})
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
t.Fatal(err)
}
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
r, rev, _ := f.runs.Get(run)
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
_, _ = f.runs.Update(run, r, rev)
}}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
return Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
}, Acts: &actLog{}}
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
if err != nil {
t.Fatalf("a rerun whose read was changed before its write: %v", err)
}
r, _, _ := f.runs.Get(run)
joined := false
for _, o := range r.Readers {
joined = joined || o.Pipeline == "main.9.app"
}
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
}
})
}
@@ -27,8 +27,8 @@ type Verbs struct {
Candidate func(group, branch string, revision int64) (Half, error)
// Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded.
Acts views.HandActs
// Drop drops the bound of an ask a cancelled pipeline no longer waits on.
Drop func(ask string) error
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
Runs RunsRead
}
func (v Verbs) now() time.Time { return v.Hearer.now() }
@@ -185,8 +185,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er
return seat.AcceptAnswer{Result: p.Steps[i]}, nil
}
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in
// the hand-act log first (§394); it says pipeline-ended.
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded
// in the hand-act log first (§394, §726); it says pipeline-ended.
func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) {
h := v.Hearer
recorded := false
@@ -205,14 +205,14 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
recorded = true
}
now := h.now()
var drop []string
var leave []seat.StepResult
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" {
drop = append(drop, s.Ask)
if sharedRun(*s) != "" && s.Seat != wants.Seat {
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
}
s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now
}
@@ -223,12 +223,8 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
} else if err != nil {
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
}
if v.Drop != nil {
for _, ask := range drop {
if err := v.Drop(ask); err != nil {
h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err)
}
}
for _, s := range leave {
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
}
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
}
@@ -242,6 +238,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
if in.Step == seat.StepResultStepApproval {
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
}
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
return v.rerunTest(caller, in, p.Steps[i])
}
}
var ask *Asked
p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error {
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
@@ -289,7 +290,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
@@ -301,6 +302,173 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil
}
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
h := v.Hearer
run := sharedRun(st)
if open(st) {
return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline)
}
r, rev, err := h.TestRuns.Get(run)
if err != nil && !errors.Is(err, ErrNoRun) {
return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err)
}
exists := err == nil
reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact}
read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended)
join := exists && r.Ended == nil
var ask *Asked
if !join && !read {
p, _, err := h.Store.Get(in.Pipeline)
if err != nil {
return seat.RerunAnswer{}, err
}
if ask, err = v.askOf(p, st, caller, h.now()); err != nil {
return seat.RerunAnswer{}, err
}
}
var asker *seat.SharedAskedBy
if join || read {
if asker, err = askerOf(h.Store, r.Ask); err != nil {
return seat.RerunAnswer{}, err
}
if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) {
asker = nil
}
}
event := seat.EventStepStarted
if read {
event = seat.EventStepEnded
}
p, _, err := v.change(in.Pipeline, event, func() error {
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
}, func(p *seat.Pipeline, now time.Time) (int, error) {
switch p.Outcome {
case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded:
return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome)
}
i, err := stepOf(*p, in.Step, in.Artifact)
if err != nil {
return -1, err
}
s := &p.Steps[i]
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now}
switch {
case ask != nil:
s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}}
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b)
case read:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now
if r.Tests != nil {
t := *r.Tests
s.Tests = &t
}
default:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
if r.Node == "" {
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
}
}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
}
for j := range p.Steps {
if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked {
p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, ""
}
}
return i, nil
})
if err != nil {
return seat.RerunAnswer{}, err
}
h.pin(p)
switch {
case join || read:
v.Starter.joinRun(run, reader)
default:
now := h.now()
a := *ask
a.Run = run
// The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and
// its readers carried, never put back; a run another step started since is said, not replaced.
for {
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
for _, o := range r.Readers {
if o.Of() == reader {
continue
}
o.Rerun, o.By = !v.passedReader(o, run), ""
if o.Rerun {
o.By = caller
}
value.Readers = append(value.Readers, o)
}
addReaders(&value, []seat.TestRunReader{reader})
err := v.Starter.started(a, func() error {
if !exists {
return h.TestRuns.Create(run, value)
}
_, err := h.TestRuns.Update(run, value, rev)
return err
})
if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) {
got, gotRev, gerr := h.TestRuns.Get(run)
switch {
case errors.Is(gerr, ErrNoRun):
r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false
continue
case gerr != nil:
err = gerr
case got.Ended == nil:
err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask)
default:
r, rev, exists = got, gotRev, true
continue
}
}
if err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
}
break
}
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
i, err := stepOf(p, in.Step, in.Artifact)
if err != nil {
return seat.RerunAnswer{}, err
}
res := p.Steps[i]
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
}
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
func (v Verbs) passedReader(reader RunReader, run string) bool {
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
if err != nil {
return true // a pipeline gone has no step to move
}
for _, s := range p.Steps {
if s.Artifact == reader.Artifact && sharedRun(s) == run {
switch s.Outcome {
case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted,
seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled:
return true
}
return s.Accepted != nil
}
}
return true
}
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
// under one id is kept once), saying it is a rerun and by whom.
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
@@ -325,7 +493,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
} else {
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
}
_, placed := Place(p, reg, half.Entries, now, v.Starter.Name)
_, placed := place(p, reg, half.Entries, now, v.Starter.Name, false)
base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step)
for _, a := range placed {
if a.ID != base {
@@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string)
asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{}
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true },
Now: h.Now}
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts,
Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped
h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil }
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts},
store, said, asks, acts, dropped
}
func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult {
@@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) {
// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again
// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends
// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394).
// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as
// A-466y amends it).
func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) {
v, store, said, asks, acts, dropped := verbsFixture()
end := func(step, outcome string) {
@@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin
}
// A cancel ends the pipeline cancelled and drops the bounds of the asks still held.
c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"})
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 {
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 {
t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped)
}
if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" {
@@ -55,9 +55,12 @@ func TestManifest(t *testing.T) {
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
equalSets(m.Claims[0].Serves, seat.Verbs())},
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})},
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})},
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
{"bounds is key-ttl", keyTTL["bounds"]},
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
{"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64},
{"the test runs' settings", runSettingsOf(raw)},
// RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB.
{"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256},
{"kept 128 MiB", stateMiB[seat.StateKept] == 128},
@@ -78,10 +81,13 @@ func TestManifest(t *testing.T) {
{"consumes", equalSets(m.Consumes, []string{
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
// §647 as A-466z amends it: no holder is read from the seat-holders view.
"seat:mesh-module-assigner.views.seat-holders"})},
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
// asked again by hand (a hand act).
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations",
@@ -107,7 +113,8 @@ func TestManifest(t *testing.T) {
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
// RN25 (§539): the runner's tools carry the prefix runner_.
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
"runner_test_runs", "runner_test_runs_drop"})},
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
// never by a tool of its own; runner_health stays a tool.
{"health is the unit", healthTool(raw) == "unit"},
@@ -175,3 +182,43 @@ func equalSets(a, b []string) bool {
slices.Sort(y)
return slices.Equal(x, y)
}
// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a
// whole number of at most 64, the three times Go durations, each written into the file its environment names.
func runSettingsOf(raw []byte) bool {
var m struct {
Config map[string]map[string]any `json:"config"`
Resources []struct {
Type string `json:"type"`
Path string `json:"path"`
Content string `json:"content"`
Env map[string]string `json:"env"`
} `json:"resources"`
}
if json.Unmarshal(raw, &m) != nil {
return false
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) {
return false
}
env := map[string]bool{}
for _, r := range m.Resources {
for _, v := range r.Env {
env[v] = true
}
}
for key, kind := range want {
if m.Config[key]["type"] != kind {
return false
}
found := false
for _, r := range m.Resources {
found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path])
}
if !found {
return false
}
}
return true
}
+82 -4
View File
@@ -18,7 +18,9 @@
"rerun",
"accept",
"cancel",
"ask-again"
"ask-again",
"test-runs",
"test-runs-drop"
]
}
],
@@ -27,7 +29,9 @@
"runner_first_view_again",
"runner_health",
"runner_unlogged",
"runner_unlogged_append"
"runner_unlogged_append",
"runner_test_runs",
"runner_test_runs_drop"
],
"consumes": [
"mesh-delivery-indexer.take-in.taken",
@@ -41,13 +45,17 @@
"node-build-go.ask-taken",
"node-build-node.ask-taken",
"node-build-image.ask-taken",
"node-build-go.step-progress",
"node-build-node.step-progress",
"node-build-image.step-progress",
"mesh-wants.step-ended",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision",
"mesh-conditions.findings-asked",
"mesh-delivery.recovery-merged",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed"
"mesh-catalog.seat-definition-removed",
"seat:mesh-module-assigner.views.seat-holders"
],
"invokes": [
"seat:mesh-catalog.seat-definitions",
@@ -79,6 +87,34 @@
"uses": [
"operator-channel"
],
"config": {
"test-runs-cap-mib": {
"type": "number",
"description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.",
"minimum": 1,
"maximum": 64,
"trusted": false,
"default": 16
},
"test-runs-kept": {
"type": "string",
"description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.",
"trusted": false,
"default": "168h"
},
"test-step-bound": {
"type": "string",
"description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).",
"trusted": false,
"default": "30m"
},
"run-untaken-bound": {
"type": "string",
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
"trusted": false,
"default": "24h"
}
},
"state": [
{
"name": "pipelines",
@@ -107,6 +143,12 @@
"history": 1,
"key-ttl": true
},
{
"name": "test-runs",
"history": 1,
"max-mib": 64,
"key-ttl": true
},
{
"name": "findings",
"history": 1
@@ -162,6 +204,38 @@
"place": ".",
"unknown": "program"
},
{
"id": "test-runs-cap",
"type": "file",
"path": "${dir:state}/test-runs-cap-mib",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-runs-cap-mib}"
},
{
"id": "test-runs-kept",
"type": "file",
"path": "${dir:state}/test-runs-kept",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-runs-kept}"
},
{
"id": "test-step-bound",
"type": "file",
"path": "${dir:state}/test-step-bound",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-step-bound}"
},
{
"id": "run-untaken-bound",
"type": "file",
"path": "${dir:state}/run-untaken-bound",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:run-untaken-bound}"
},
{
"id": "pipeline",
"type": "process",
@@ -171,7 +245,11 @@
"./mesh-delivery-pipeline"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker"
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib",
"MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept",
"MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound",
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound"
},
"user": "mesh-delivery-pipeline",
"health": {