Merge branch 'fix/47-G-shared-runs-runner' into fix/47-G-shared-runs
This commit is contained in:
@@ -46,7 +46,7 @@ func as[T any](args map[string]any) (T, error) {
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// Tools are the runner's read verbs over its states pipelines and kept.
|
||||
// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i).
|
||||
func Tools(r pipeline.Reads) []stdio.Tool {
|
||||
runs := map[string]func(map[string]any) (any, error){
|
||||
seat.VerbShow: func(a map[string]any) (any, error) {
|
||||
@@ -152,6 +152,7 @@ func main() {
|
||||
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
|
||||
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
|
||||
tools = append(tools, TestRunsTools(stdio.Ask)...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb
|
||||
// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i).
|
||||
func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: Prefix + "_test_runs",
|
||||
Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"},
|
||||
"digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[seat.TestRunsInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in)
|
||||
}},
|
||||
{Name: Prefix + "_test_runs_drop",
|
||||
Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[seat.TestRunsDropInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" {
|
||||
return nil, errors.New("test-runs-drop names the run's key and why")
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in)
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string
|
||||
// armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222).
|
||||
func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error {
|
||||
return func(key string) error {
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound))
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
@@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
|
||||
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask),
|
||||
Headline: "A call whose answer is kept is not answered",
|
||||
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound),
|
||||
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound),
|
||||
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}})
|
||||
}
|
||||
}
|
||||
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no
|
||||
// holder took (§32, §535, A-77); another key is none of the runner's.
|
||||
func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
|
||||
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
|
||||
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
return func(key string) {
|
||||
if s, ask, ok := pipeline.ParseDropBound(key); ok {
|
||||
if err := unanswered(s, ask); err != nil {
|
||||
logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !pipeline.IsAskKey(key) {
|
||||
return
|
||||
}
|
||||
if err := h.NotTaken(key, pipeline.AskBound); err != nil {
|
||||
logf("the step of ask %s, which no holder took, is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error
|
||||
func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
|
||||
w := &fakeFindings{}
|
||||
var logged int
|
||||
expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
|
||||
expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
|
||||
expired("not-a-bound")
|
||||
if len(w.written) != 0 {
|
||||
t.Fatalf("a key of no drop was said: %+v", w.written)
|
||||
|
||||
@@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
|
||||
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// for its step.
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
|
||||
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
|
||||
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
|
||||
return err
|
||||
}
|
||||
resume := func(branch string) {
|
||||
rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647)
|
||||
half, ok := rv.Catalog.View(branch)
|
||||
if !ok {
|
||||
return
|
||||
@@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
resumer.Store(&resume)
|
||||
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
|
||||
advance := func(branch string) {
|
||||
rv.Holders.refresh()
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
@@ -227,23 +229,41 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31,
|
||||
// §350; A-182, A-202).
|
||||
starter.Supersede = func(newer seat.Pipeline) {
|
||||
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds))
|
||||
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)))
|
||||
if err != nil {
|
||||
logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err)
|
||||
} else if n > 0 {
|
||||
logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n)
|
||||
}
|
||||
}
|
||||
// The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock.
|
||||
runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))
|
||||
runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearStepEnded(heard, hearer, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
|
||||
taken := func(ask, node string) error {
|
||||
if err := hearer.Taken(ask, node); err != nil {
|
||||
return err
|
||||
}
|
||||
return hearer.RunTaken(ask, node)
|
||||
}
|
||||
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -261,6 +281,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// An entry whose step ended and was not appended is appended at the start, in order (§606, §549).
|
||||
hearer.AppendUnlogged()
|
||||
// A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725).
|
||||
if n, err := hearer.TakeUp(); err != nil || n > 0 {
|
||||
logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err)
|
||||
}
|
||||
go whileTheLogIsFull(ctx, fills, hearer, time.Minute)
|
||||
// The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check
|
||||
// their callers against (A-186): registered before the hub starts, read again once its events are live.
|
||||
@@ -273,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
|
||||
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
|
||||
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
|
||||
stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds))
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
stopVerbs, err := answerVerbs(calls, seats, verbs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
|
||||
for _, state := range []string{"bounds", TestRunsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
|
||||
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
_, err = h.OnStepEnded(bs, m.Subject(), ev)
|
||||
var shared []seat.Shared
|
||||
if bs.Shared != nil {
|
||||
shared = bs.Shared(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrRefused):
|
||||
logf("%v", err)
|
||||
@@ -133,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
return nil
|
||||
}
|
||||
|
||||
// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took
|
||||
// it (A-182): the ask is answered, so its five minutes end with no finding.
|
||||
func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
|
||||
// that node's holder, and begins the clock of a test run's ask (§725).
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
for _, ts := range pipeline.TakenSeats {
|
||||
ts := ts
|
||||
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
@@ -145,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
// The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182).
|
||||
if err := taken(ev.Ask, ev.Node); err != nil {
|
||||
logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err)
|
||||
_ = m.Nak()
|
||||
@@ -165,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
|
||||
return nil
|
||||
}
|
||||
|
||||
// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask.
|
||||
func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
|
||||
return func(ask string) error {
|
||||
_, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
|
||||
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
|
||||
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
|
||||
// unanswered (A-222, A-77): no timer in the process.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section.
|
||||
const TestRunsState = "test-runs"
|
||||
|
||||
// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own
|
||||
// time to live (A-77, A-470).
|
||||
type testRunsKV struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (t testRunsKV) Create(key string, r pipeline.Run) error {
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return pipeline.ErrExists
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) {
|
||||
e, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return pipeline.Run{}, 0, pipeline.ErrNoRun
|
||||
}
|
||||
if err != nil {
|
||||
return pipeline.Run{}, 0, err
|
||||
}
|
||||
var r pipeline.Run
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err)
|
||||
}
|
||||
return r, e.Revision(), nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) {
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := t.kv.Update(t.ctx, key, raw, revision)
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Delete(key string, revision uint64) error {
|
||||
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return pipeline.ErrChanged
|
||||
}
|
||||
if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteBound deletes a bound key, whose value is no run: a key gone is no fault.
|
||||
func (t testRunsKV) deleteBound(key string) error {
|
||||
if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject := "$KV." + t.kv.Bucket() + "." + key
|
||||
for {
|
||||
var last uint64
|
||||
st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m, err := st.GetLastMsgForSubject(t.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Bounded reads a bound key: the ask it holds and its revision.
|
||||
func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) {
|
||||
e, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
return v.Ask, e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
|
||||
|
||||
// UnboundAt deletes a bound key at revision: a later write of it is kept.
|
||||
func (t testRunsKV) UnboundAt(key string, revision uint64) error {
|
||||
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Has(key string) (bool, error) {
|
||||
_, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Keys() ([]string, error) {
|
||||
all, err := t.kv.Keys(t.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return all, err
|
||||
}
|
||||
|
||||
// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written.
|
||||
type runsKept struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (r runsKept) Keys() ([]string, error) {
|
||||
ks, err := r.kv.Keys(r.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return ks, err
|
||||
}
|
||||
|
||||
func (r runsKept) Value(key string) (json.RawMessage, error) {
|
||||
e, err := r.kv.Get(r.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value(), nil
|
||||
}
|
||||
|
||||
// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when
|
||||
// first used.
|
||||
type runObjects struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
mu sync.Mutex
|
||||
store jetstream.ObjectStore
|
||||
}
|
||||
|
||||
func (o *runObjects) open() (jetstream.ObjectStore, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.store != nil {
|
||||
return o.store, nil
|
||||
}
|
||||
s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err)
|
||||
}
|
||||
o.store = s
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (o *runObjects) Put(name string, data []byte) error {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.PutBytes(o.ctx, name, data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *runObjects) Get(name string) ([]byte, error) {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := s.GetBytes(o.ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return nil, pipeline.ErrNoRun
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (o *runObjects) Delete(name string) error {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting.
|
||||
func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) {
|
||||
return func() (seat.Fill, error) {
|
||||
st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState))
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
info, err := st.Info(ctx)
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
at := now().UTC()
|
||||
return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// The files the manifest writes each setting of the test runs into, `${config:<key>}`, named by the environment (A-469x).
|
||||
const (
|
||||
envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE"
|
||||
envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE"
|
||||
envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE"
|
||||
envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE"
|
||||
)
|
||||
|
||||
// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long.
|
||||
const maxRunsKept = 168 * time.Hour
|
||||
|
||||
// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB,
|
||||
// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its
|
||||
// default; a content that is neither is an error naming the file.
|
||||
func runSettings(getenv func(string) string) (pipeline.RunSettings, error) {
|
||||
s := pipeline.DefaultRunSettings
|
||||
if v, err := settingText(getenv(envRunsCap)); err != nil {
|
||||
return s, err
|
||||
} else if v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v)
|
||||
}
|
||||
s.CapBytes = int64(n) << 20
|
||||
}
|
||||
for _, d := range []struct {
|
||||
env string
|
||||
to *time.Duration
|
||||
}{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} {
|
||||
v, err := settingText(getenv(d.env))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
t, err := time.ParseDuration(v)
|
||||
if err != nil || t <= 0 {
|
||||
return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v)
|
||||
}
|
||||
*d.to = t
|
||||
}
|
||||
if s.Kept > maxRunsKept {
|
||||
return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one.
|
||||
func settingText(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// progressSubjects hear every build seat's step-progress (§725), from their contracts.
|
||||
var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll}
|
||||
|
||||
// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p).
|
||||
func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error {
|
||||
for _, subject := range progressSubjects {
|
||||
err := hub.Handle(subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.StepProgress
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
logf("%s is no step-progress: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := progress(ev.Ask); err != nil {
|
||||
logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the
|
||||
// process.
|
||||
func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = w.Stop()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) {
|
||||
continue
|
||||
}
|
||||
if err := expired(e.Key()); err != nil {
|
||||
logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725).
|
||||
func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string,
|
||||
starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) {
|
||||
settings, err := runSettings(getenv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
runs := testRunsKV{ctx: ctx, js: js, kv: kv}
|
||||
objects := &runObjects{ctx: ctx, js: js, module: cred.Module}
|
||||
starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings
|
||||
starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now)
|
||||
starter.EndReaders = hearer.EndReaders
|
||||
hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop
|
||||
hearer.RunOutputs = objects
|
||||
if hearer.Ager != nil {
|
||||
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept
|
||||
}
|
||||
logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken)
|
||||
return kv, objects, nil
|
||||
}
|
||||
+536
@@ -0,0 +1,536 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in
|
||||
// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a
|
||||
// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps
|
||||
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
|
||||
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
|
||||
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
|
||||
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
|
||||
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
|
||||
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
|
||||
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
|
||||
// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill.
|
||||
// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes.
|
||||
func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const module = "mesh-delivery-pipeline"
|
||||
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, state := range []string{"bounds", TestRunsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// test-runs' own object store, made with the state (A-469e).
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go
|
||||
// has a holder, node-build-node none.
|
||||
holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module)
|
||||
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
|
||||
// integration step optional; web is built by node-build-node.
|
||||
policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}}
|
||||
regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy},
|
||||
{Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}}
|
||||
answer := func(subject string, body any) {
|
||||
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": body})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
|
||||
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
|
||||
entries := []catalog.CatalogEntry{
|
||||
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
|
||||
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
|
||||
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}},
|
||||
{Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}},
|
||||
}
|
||||
var revision atomic.Int64
|
||||
revision.Store(1)
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
|
||||
r := revision.Load()
|
||||
raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
|
||||
var in struct {
|
||||
Seat string `json:"seat"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
|
||||
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
|
||||
if in.Seat != "" {
|
||||
entry.Definition = def
|
||||
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": result})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Every pipeline-ended said, and every drop asked of a build seat.
|
||||
var mu sync.Mutex
|
||||
var endings []seat.PipelineEnded
|
||||
var drops []string
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) {
|
||||
var e seat.PipelineEnded
|
||||
if json.Unmarshal(m.Data, &e) == nil {
|
||||
mu.Lock()
|
||||
endings = append(endings, e)
|
||||
mu.Unlock()
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
drops = append(drops, string(m.Data))
|
||||
mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{}})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := func(key string) []seat.PipelineEnded {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
var out []seat.PipelineEnded
|
||||
for _, e := range endings {
|
||||
if e.Pipeline == key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds.
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{envRunsCap: "1", envStepBound: "2s"}
|
||||
env := map[string]string{}
|
||||
for name, content := range files {
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = path
|
||||
}
|
||||
broker := filepath.Join(dir, "broker")
|
||||
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
|
||||
if err := os.WriteFile(broker, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[process.BrokerFileEnv] = broker
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stop := start()
|
||||
defer func() { stop() }()
|
||||
|
||||
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders"))
|
||||
var s views.Status
|
||||
return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero()
|
||||
})
|
||||
|
||||
takeIn := func(module, commit, fp string, source, rev int64) {
|
||||
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev,
|
||||
Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
placed := func(key string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 })
|
||||
return p
|
||||
}
|
||||
read := func(key string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
readPipeline(ctx, pipelines, key, &p)
|
||||
return p
|
||||
}
|
||||
step := func(p seat.Pipeline, artifact, name string) seat.StepResult {
|
||||
for _, s := range p.Steps {
|
||||
if s.Artifact == artifact && s.Step == name {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return seat.StepResult{}
|
||||
}
|
||||
runOf := func(key string) (pipeline.Run, bool) {
|
||||
e, err := runsKV.Get(ctx, key)
|
||||
var r pipeline.Run
|
||||
return r, err == nil && json.Unmarshal(e.Value(), &r) == nil
|
||||
}
|
||||
runKeys := func(candidate string) []string {
|
||||
ks, _ := runsKV.Keys(ctx)
|
||||
var out []string
|
||||
for _, k := range ks {
|
||||
if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
runOfStep := func(key, artifact, name string) (string, pipeline.Run) {
|
||||
var run string
|
||||
var r pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
s := step(read(key), artifact, name)
|
||||
if len(s.Shared) == 0 {
|
||||
return false
|
||||
}
|
||||
run = s.Shared[0].Run
|
||||
got, ok := runOf(run)
|
||||
r = got
|
||||
return ok && r.Ask != ""
|
||||
})
|
||||
return run, r
|
||||
}
|
||||
publish := func(subject string, body any) {
|
||||
raw, _ := json.Marshal(body)
|
||||
if _, err := js.Publish(ctx, subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
taken := func(ask string) {
|
||||
publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()})
|
||||
}
|
||||
endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) {
|
||||
now := time.Now()
|
||||
publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1",
|
||||
Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now})
|
||||
}
|
||||
passBuild := func(key, module, commit, fp string) {
|
||||
ask := step(read(key), "", seat.StepResultStepBuild).Ask
|
||||
publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
|
||||
Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
|
||||
}
|
||||
call := func(verb string, in, out any) string {
|
||||
var reply struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second)
|
||||
reply.Error = ""
|
||||
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
|
||||
})
|
||||
if out != nil && reply.Error == "" {
|
||||
_ = json.Unmarshal(reply.Result, out)
|
||||
}
|
||||
return reply.Error
|
||||
}
|
||||
|
||||
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
|
||||
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
|
||||
const p3 = "main.3.app"
|
||||
takeIn("app", "c1", "src2:aa", 3, 1)
|
||||
passBuild(p3, "app", "c1", "src2:aa")
|
||||
unitRun, unit := runOfStep(p3, "app:program", "unit")
|
||||
integRun, integ := runOfStep(p3, "app:program", "integration")
|
||||
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
|
||||
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
|
||||
}
|
||||
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
|
||||
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
|
||||
}
|
||||
|
||||
// A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both.
|
||||
taken(unit.Ask)
|
||||
waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" })
|
||||
// Its clock holds the run's ask (A-470n).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
|
||||
return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`)
|
||||
})
|
||||
stop()
|
||||
stop = start()
|
||||
// It keeps saying its progress past its bound and is not ended (§725, §720).
|
||||
// Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises).
|
||||
boundRev := func() uint64 {
|
||||
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return e.Revision()
|
||||
}
|
||||
for at := time.Now(); time.Since(at) < 3*time.Second; {
|
||||
before := boundRev()
|
||||
publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()})
|
||||
waitFor(t, ctx, logs, func() bool { return boundRev() > before })
|
||||
}
|
||||
if r, _ := runOf(unitRun); r.Ended != nil {
|
||||
t.Fatalf("a run saying its progress past its bound ended: %+v", r)
|
||||
}
|
||||
// A failing test fails once and is counted once (§726).
|
||||
endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5})
|
||||
taken(integ.Ask)
|
||||
endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2})
|
||||
waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 })
|
||||
p := read(p3)
|
||||
for _, a := range []string{"app:program", "app:tools"} {
|
||||
if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun {
|
||||
t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s)
|
||||
}
|
||||
}
|
||||
for _, k := range []string{unitRun, integRun} {
|
||||
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed {
|
||||
t.Errorf("the run %s at its end = %+v; want it ended failed", k, r)
|
||||
}
|
||||
}
|
||||
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 {
|
||||
t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e)
|
||||
}
|
||||
|
||||
// An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the
|
||||
// pipeline says pipeline-ended again (§727, A-470d).
|
||||
if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" {
|
||||
t.Fatalf("accept: %s", e)
|
||||
}
|
||||
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" {
|
||||
t.Fatalf("rerun: %s", e)
|
||||
}
|
||||
var again pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask })
|
||||
taken(again.Ask)
|
||||
endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5})
|
||||
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" {
|
||||
t.Fatalf("rerun: %s", e)
|
||||
}
|
||||
var integAgain pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
r, ok := runOf(integRun)
|
||||
integAgain = r
|
||||
return ok && r.Ended == nil && r.Ask != integ.Ask
|
||||
})
|
||||
taken(integAgain.Ask)
|
||||
endAsk(p3, integAgain.Ask, "integration", "passed", nil)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e := ended(p3)
|
||||
return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed
|
||||
})
|
||||
p = read(p3)
|
||||
if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun {
|
||||
t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s)
|
||||
}
|
||||
if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted {
|
||||
t.Errorf("the accepted step = %+v; want it accepted still", s)
|
||||
}
|
||||
for _, k := range []string{unitRun, integRun} {
|
||||
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed {
|
||||
t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r)
|
||||
}
|
||||
}
|
||||
if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount {
|
||||
t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount)
|
||||
}
|
||||
|
||||
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
|
||||
revision.Store(2)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
|
||||
const p4, p5 = "main.4.app", "main.5.app"
|
||||
takeIn("app", "c1", "src2:aa", 4, 2)
|
||||
placed(p4)
|
||||
run4, r4 := runOfStep(p4, "app:program", "integration")
|
||||
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
|
||||
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
|
||||
}
|
||||
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
|
||||
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
|
||||
takeIn("app", "c1", "src2:aa", 5, 2)
|
||||
placed(p5)
|
||||
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
|
||||
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
|
||||
t.Fatalf("cancel: %s", e)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
r, _ := runOf(run4)
|
||||
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
|
||||
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := append([]string(nil), drops...)
|
||||
mu.Unlock()
|
||||
if len(dropped) != 0 {
|
||||
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
|
||||
}
|
||||
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
|
||||
taken(r4.Ask)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
|
||||
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
|
||||
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(drops) == 1 && strings.Contains(drops[0], r4.Ask)
|
||||
})
|
||||
|
||||
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
|
||||
// asked (§535, §647; A-470g, A-470h).
|
||||
const web = "main.5.web"
|
||||
takeIn("web", "w1", "src2:ww", 5, 2)
|
||||
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
|
||||
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
|
||||
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
|
||||
}
|
||||
// The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked,
|
||||
// and none is asked (A-470h).
|
||||
for _, s := range read(web).Steps {
|
||||
want := seat.StepResultOutcomeNotAsked
|
||||
switch {
|
||||
case s.Status == seat.StepResultStatusBypass:
|
||||
want = seat.StepResultOutcomeBypassed
|
||||
case s.Seat == "node-build-node":
|
||||
want = seat.StepResultOutcomeCouldNotRun
|
||||
}
|
||||
if s.Outcome != want || s.Ask != "" {
|
||||
t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want)
|
||||
}
|
||||
}
|
||||
|
||||
// test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output
|
||||
// object; test-runs-drop deletes the key and its object (§728).
|
||||
var runs seat.TestRunsAnswer
|
||||
if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState {
|
||||
t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e)
|
||||
}
|
||||
objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := objects.GetInfo(ctx, unitRun); err != nil {
|
||||
t.Fatalf("the unit run's output object: %v", err)
|
||||
}
|
||||
var drop seat.TestRunsDropAnswer
|
||||
if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun {
|
||||
t.Fatalf("test-runs-drop = %+v (%s)", drop, e)
|
||||
}
|
||||
if _, ok := runOf(unitRun); ok {
|
||||
t.Error("the dropped run's key is kept")
|
||||
}
|
||||
if _, err := objects.GetInfo(ctx, unitRun); err == nil {
|
||||
t.Error("the dropped run's output object is kept")
|
||||
}
|
||||
|
||||
// At the cap a step ends could not run, naming the fill (§725, A-469d).
|
||||
pad := make([]byte, 600<<10)
|
||||
for _, k := range []string{"pad.a", "pad.b"} {
|
||||
if _, err := runsKV.Put(ctx, k, pad); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
revision.Store(3)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
|
||||
const p6 = "main.6.app"
|
||||
takeIn("app", "c1", "src2:aa", 6, 3)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
s := step(read(p6), "app:program", "unit")
|
||||
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
|
||||
})
|
||||
}
|
||||
|
||||
func mustJSON(v any) []byte {
|
||||
raw, _ := json.Marshal(v)
|
||||
return raw
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the
|
||||
// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time
|
||||
// past 168h, an error naming the file.
|
||||
func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, content string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
rows := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
ok bool
|
||||
check func(t *testing.T, cap int64, kept, step, untaken time.Duration)
|
||||
}{
|
||||
{"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
|
||||
if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour {
|
||||
t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken)
|
||||
}
|
||||
}},
|
||||
{"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true,
|
||||
func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
|
||||
if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute {
|
||||
t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken)
|
||||
}
|
||||
}},
|
||||
{"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil},
|
||||
{"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil},
|
||||
{"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil},
|
||||
}
|
||||
for _, r := range rows {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
env := map[string]string{}
|
||||
for k, v := range r.files {
|
||||
env[k] = write(r.name+k, v)
|
||||
}
|
||||
s, err := runSettings(func(k string) string { return env[k] })
|
||||
if (err == nil) != r.ok {
|
||||
t.Fatalf("runSettings = %+v, %v", s, err)
|
||||
}
|
||||
if r.check != nil {
|
||||
r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -41,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
|
||||
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
|
||||
}
|
||||
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and
|
||||
// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i).
|
||||
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
|
||||
@@ -51,24 +49,17 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
|
||||
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
|
||||
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
|
||||
}
|
||||
}
|
||||
|
||||
// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on.
|
||||
func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error {
|
||||
return func(ask string) error {
|
||||
if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
|
||||
seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }),
|
||||
// A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against.
|
||||
seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }),
|
||||
}
|
||||
}
|
||||
|
||||
// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts.
|
||||
func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool),
|
||||
calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs {
|
||||
func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs {
|
||||
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
|
||||
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
|
||||
}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
|
||||
got = append(got, v)
|
||||
}
|
||||
slices.Sort(got)
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("the process answers %v, want %v", got, want)
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
@@ -26,6 +28,70 @@ type runnerViews struct {
|
||||
// Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own
|
||||
// findings (A-206).
|
||||
Findings *findingsWriter
|
||||
// Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z).
|
||||
Holders *holderView
|
||||
}
|
||||
|
||||
// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat
|
||||
// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders`
|
||||
// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's.
|
||||
// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances.
|
||||
type holderView struct {
|
||||
reader *views.ProcessReader
|
||||
catalog *pipeline.Views
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
owner string
|
||||
pic *views.Picture
|
||||
}
|
||||
|
||||
// refresh follows the claimant's seat-holders view when the claimant is another than the one followed.
|
||||
func (h *holderView) refresh() {
|
||||
var halves []pipeline.Half
|
||||
for _, b := range h.catalog.Branches() {
|
||||
if half, ok := h.catalog.View(b); ok {
|
||||
halves = append(halves, half)
|
||||
}
|
||||
}
|
||||
claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !ok || claimant == h.owner {
|
||||
return
|
||||
}
|
||||
if h.owner != "" {
|
||||
if err := h.reader.Drop(h.owner, seatHoldersState); err != nil {
|
||||
h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err)
|
||||
}
|
||||
h.owner, h.pic = "", nil
|
||||
}
|
||||
pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
|
||||
if err != nil {
|
||||
h.logf("the seat-holders view of %s is not followed: %v", claimant, err)
|
||||
return
|
||||
}
|
||||
h.owner, h.pic = claimant, pic
|
||||
h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat)
|
||||
}
|
||||
|
||||
// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699).
|
||||
const seatHoldersState = "seat-holders"
|
||||
|
||||
// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with
|
||||
// no claimant followed, nothing is judged (A-466y).
|
||||
func (h *holderView) Holders(seatName string) ([]string, bool) {
|
||||
h.mu.Lock()
|
||||
pic := h.pic
|
||||
h.mu.Unlock()
|
||||
if pic == nil {
|
||||
return nil, false
|
||||
}
|
||||
select {
|
||||
case <-pic.Follower().Ready():
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true
|
||||
}
|
||||
|
||||
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
|
||||
@@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
|
||||
return nil, err
|
||||
}
|
||||
reader.Start(ctx, js, cred, status)
|
||||
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil
|
||||
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
|
||||
}
|
||||
|
||||
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
|
||||
|
||||
@@ -9,6 +9,8 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -53,7 +54,12 @@ type Ager struct {
|
||||
Get(key string) (seat.KeptResult, bool, error)
|
||||
Delete(key string) error
|
||||
}
|
||||
Now func() time.Time
|
||||
// Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting
|
||||
// test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a).
|
||||
Runs TestRuns
|
||||
RunOutputs func(name string) error
|
||||
RunsKept time.Duration
|
||||
Now func() time.Time
|
||||
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
@@ -69,7 +75,16 @@ func (a *Ager) Age() {
|
||||
if a == nil || a.Log == nil {
|
||||
return
|
||||
}
|
||||
for _, r := range []Retention{Retention7, Retention30, Retention90} {
|
||||
retentions := []Retention{Retention7, Retention30, Retention90}
|
||||
if a.Runs != nil {
|
||||
// The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a).
|
||||
kept := a.RunsKept
|
||||
if kept <= 0 {
|
||||
kept = DefaultRunSettings.Kept
|
||||
}
|
||||
retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...)
|
||||
}
|
||||
for _, r := range retentions {
|
||||
if err := a.ageAt(r); err != nil {
|
||||
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
|
||||
}
|
||||
@@ -103,8 +118,57 @@ func (a *Ager) ageAt(r Retention) error {
|
||||
return a.Aged.Put(r.Name, moved)
|
||||
}
|
||||
|
||||
// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725).
|
||||
const RetentionTestRuns = "test-runs"
|
||||
|
||||
// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the
|
||||
// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run.
|
||||
func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
|
||||
if e.What != seat.LogEntryWhatEnded {
|
||||
return
|
||||
}
|
||||
p, _, err := a.Pipelines.Get(e.Pipeline)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
run := ""
|
||||
for _, s := range p.Steps {
|
||||
if s.Step == e.Step && s.Artifact == e.Artifact {
|
||||
run = sharedRun(s)
|
||||
}
|
||||
}
|
||||
if run == "" {
|
||||
return
|
||||
}
|
||||
for {
|
||||
t, rev, err := a.Runs.Get(run)
|
||||
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
|
||||
// ended yet is aged at a later reader's end.
|
||||
if err != nil || t.Ended == nil {
|
||||
return
|
||||
}
|
||||
if t.Output != "" && a.RunOutputs != nil {
|
||||
if err := a.RunOutputs(t.Output); err != nil {
|
||||
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept.
|
||||
if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
a.logf("aged: the test run %s is not deleted: %v", run, err)
|
||||
return
|
||||
}
|
||||
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
|
||||
switch r.Name {
|
||||
case RetentionTestRuns:
|
||||
a.ageRun(r, e)
|
||||
case Retention7.Name:
|
||||
if e.Output != "" && a.Outputs != nil {
|
||||
if err := a.Outputs(e.Output); err != nil {
|
||||
|
||||
@@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
|
||||
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
|
||||
return -1
|
||||
}
|
||||
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||
)
|
||||
|
||||
// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five
|
||||
// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer.
|
||||
const AskBound = 5 * time.Minute
|
||||
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
|
||||
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
|
||||
const DropBound = 5 * time.Minute
|
||||
|
||||
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
|
||||
type TakenSeat struct {
|
||||
@@ -23,15 +22,15 @@ type TakenSeat struct {
|
||||
Subject string
|
||||
}
|
||||
|
||||
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken,
|
||||
// so no bound is armed for their asks.
|
||||
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken.
|
||||
var TakenSeats = []TakenSeat{
|
||||
{Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll},
|
||||
{Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll},
|
||||
{Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll},
|
||||
}
|
||||
|
||||
// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound.
|
||||
// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share
|
||||
// a run (§724).
|
||||
func BoundSeat(name string) bool {
|
||||
for _, s := range TakenSeats {
|
||||
if s.Seat == name {
|
||||
@@ -41,63 +40,6 @@ func BoundSeat(name string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own
|
||||
// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it.
|
||||
type Arm func(ask string) error
|
||||
|
||||
// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32,
|
||||
// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never
|
||||
// a silent wait. A step that ended or was never asked of a seat takes nothing.
|
||||
func (h *Hearer) NotTaken(ask string, bound time.Duration) error {
|
||||
pkeys, err := h.Store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := h.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 {
|
||||
continue
|
||||
}
|
||||
return h.endNotTaken(key, ask, bound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now
|
||||
s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound)
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
|
||||
ended := endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
h.Ager.Age()
|
||||
h.Fills.Check()
|
||||
return h.said(p, false, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// IsAskKey says whether a key of the state `bounds` is an ask's bound.
|
||||
func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") }
|
||||
|
||||
// DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop.<build seat>.<ask>` (design 47 G
|
||||
// A-222, A-77): its five-minute expiry says the drop unanswered.
|
||||
const DropBoundPrefix = "drop."
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
|
||||
}
|
||||
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
|
||||
said = nil
|
||||
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
|
||||
t.Fatalf("supersede %d (%v)", n, err)
|
||||
}
|
||||
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
|
||||
|
||||
@@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
|
||||
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
|
||||
// work queues, one per running step.
|
||||
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
|
||||
return place(p, reg, entries, now, asker, true)
|
||||
}
|
||||
|
||||
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
|
||||
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
|
||||
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
|
||||
// naming the run's ask and the step that asked it (A-469m, A-469q).
|
||||
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
|
||||
entry := entries[p.Module]
|
||||
candidate, named := CandidateOf(p)
|
||||
share = share && named
|
||||
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
|
||||
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
|
||||
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
|
||||
var steps []seat.StepResult
|
||||
@@ -118,8 +129,27 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
|
||||
}
|
||||
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
|
||||
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
|
||||
if share && BoundSeat(bs) {
|
||||
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
|
||||
run := TestRunKey(candidate, step, digest)
|
||||
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
|
||||
if j, asked := askOfRun[run]; asked {
|
||||
// Another artifact's step of this pipeline asks the run: this one reads it.
|
||||
first := &asks[j]
|
||||
first.Readers = append(first.Readers, reader)
|
||||
s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
// A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b).
|
||||
s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
|
||||
askOfRun[run] = len(asks)
|
||||
}
|
||||
steps = append(steps, s)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
|
||||
if a.Package != nil {
|
||||
ask.Package = a.Package.Name
|
||||
}
|
||||
@@ -156,6 +186,10 @@ type Asked struct {
|
||||
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
|
||||
// are the ask's (§25, §416).
|
||||
Package string
|
||||
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
|
||||
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
|
||||
Run, Level, Digest string
|
||||
Readers []seat.TestRunReader
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
@@ -170,8 +204,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step,
|
||||
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put.
|
||||
func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error {
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
@@ -179,15 +214,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e
|
||||
accept = to
|
||||
}
|
||||
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
if ok {
|
||||
// A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647).
|
||||
if arm != nil && BoundSeat(a.Seat) {
|
||||
if aerr := arm(a.ID); aerr != nil {
|
||||
err = fmt.Errorf("its bound is not armed: %w", aerr)
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
|
||||
@@ -328,3 +328,207 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
|
||||
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
|
||||
}
|
||||
}
|
||||
|
||||
// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read.
|
||||
type testRunsMem map[string]struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}
|
||||
|
||||
func (m testRunsMem) Create(key string, r Run) error {
|
||||
if _, ok := m[key]; ok {
|
||||
return ErrExists
|
||||
}
|
||||
m[key] = struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}{r, 1}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Get(key string) (Run, uint64, error) {
|
||||
e, ok := m[key]
|
||||
if !ok {
|
||||
return Run{}, 0, ErrNoRun
|
||||
}
|
||||
e.r.Readers = append([]RunReader(nil), e.r.Readers...)
|
||||
return e.r, e.rev, nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Delete(key string, rev uint64) error {
|
||||
if e, ok := m[key]; ok && e.rev != rev {
|
||||
return ErrChanged
|
||||
}
|
||||
delete(m, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) {
|
||||
if m[key].rev != rev {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
m[key] = struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}{r, rev + 1}
|
||||
return rev + 1, nil
|
||||
}
|
||||
|
||||
// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle
|
||||
// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second
|
||||
// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same
|
||||
// digests asks its own runs.
|
||||
func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{}
|
||||
s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks,
|
||||
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
|
||||
start := func(source, revision int64) seat.Pipeline {
|
||||
t.Helper()
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start %d = %v, %v", source, ok, err)
|
||||
}
|
||||
p, _, _ := store.Get(Key("main", source, "app"))
|
||||
return p
|
||||
}
|
||||
testAsks := func(from int) []string {
|
||||
var out []string
|
||||
for _, a := range asks.asks[from:] {
|
||||
if b, ok := a.(nodebuildgo.StepAsk); ok {
|
||||
out = append(out, b.Step.Artifact+" "+b.Step.Step)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
p := start(4, 9)
|
||||
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
|
||||
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
|
||||
}
|
||||
for _, level := range buildSteps {
|
||||
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
|
||||
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
|
||||
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
|
||||
}
|
||||
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
|
||||
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
|
||||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
|
||||
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
|
||||
}
|
||||
r, _, err := runs.Get(asker.Shared[0].Run)
|
||||
if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") {
|
||||
t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err)
|
||||
}
|
||||
}
|
||||
// A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs.
|
||||
n := len(asks.asks)
|
||||
q := start(5, 9)
|
||||
if got := testAsks(n); len(got) != 0 {
|
||||
t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got)
|
||||
}
|
||||
unit := stepNamed(q, "app:program", "unit")
|
||||
if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
|
||||
t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit)
|
||||
}
|
||||
if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 {
|
||||
t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers)
|
||||
}
|
||||
// A run ended could not run is never read: the next step of its key asks it again.
|
||||
key := stepNamed(p, "app:program", "lint").Shared[0].Run
|
||||
r, rev, _ := runs.Get(key)
|
||||
ended := time.Unix(9, 0)
|
||||
r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun
|
||||
_, _ = runs.Update(key, r, rev)
|
||||
n = len(asks.asks)
|
||||
start(6, 9)
|
||||
if got := strings.Join(testAsks(n), "|"); got != "app:program lint" {
|
||||
t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got)
|
||||
}
|
||||
// A second revision with the same digests asks its own runs, so a failed test runs again.
|
||||
n = len(asks.asks)
|
||||
start(7, 10)
|
||||
if got := len(testAsks(n)); got != 3 {
|
||||
t.Errorf("a second revision asked %d test steps, want its own three", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them):
|
||||
// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat,
|
||||
// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h);
|
||||
// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of
|
||||
// the view is the one module the catalog halves name.
|
||||
func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
holders []string
|
||||
read bool
|
||||
asked bool
|
||||
}{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
store, asks := memStore{}, &stepAsks{}
|
||||
var said []int
|
||||
s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
|
||||
Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Holders: func(string) ([]string, bool) { return c.holders, c.read },
|
||||
Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }}
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := store.Get("main.4.app")
|
||||
lint := stepNamed(p, "app:program", "lint")
|
||||
asked := false
|
||||
for _, id := range asks.ids {
|
||||
asked = asked || id == lint.Ask
|
||||
}
|
||||
if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) {
|
||||
t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked)
|
||||
}
|
||||
if c.asked {
|
||||
if p.Outcome != seat.PipelineOutcomeRunning || said != nil {
|
||||
t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 {
|
||||
t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids))
|
||||
}
|
||||
if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 {
|
||||
t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want)
|
||||
}
|
||||
ended := 0
|
||||
for _, st := range p.Steps {
|
||||
switch {
|
||||
case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked:
|
||||
ended++
|
||||
case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "":
|
||||
// ended by Place itself: a bypass, or a language no build seat names
|
||||
default:
|
||||
t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome)
|
||||
}
|
||||
}
|
||||
if ended < 3 || len(said) != ended {
|
||||
t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said))
|
||||
}
|
||||
})
|
||||
}
|
||||
claims := func(seats ...string) map[string]any {
|
||||
var cs []any
|
||||
for _, n := range seats {
|
||||
cs = append(cs, map[string]any{"name": n, "scope": "mesh"})
|
||||
}
|
||||
return map[string]any{"module": "x", "version": "0.1", "claims": cs}
|
||||
}
|
||||
half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")},
|
||||
"app": {Module: "app", Manifest: claims()}}}
|
||||
if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" {
|
||||
t.Errorf("the claimant = %q, %v", got, ok)
|
||||
}
|
||||
other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}}
|
||||
if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok {
|
||||
t.Errorf("halves that disagree name %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,12 +95,12 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
|
||||
if s.Ended != nil || s.Started == nil {
|
||||
continue
|
||||
}
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
|
||||
Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step,
|
||||
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1})
|
||||
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
|
||||
@@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time {
|
||||
}
|
||||
return *p.Started
|
||||
}
|
||||
|
||||
// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m).
|
||||
func askOfWait(s seat.StepResult) string { return s.Ask }
|
||||
|
||||
// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key
|
||||
// and ask, as one entry with no machine and no asker.
|
||||
func sharedOfWait(s seat.StepResult) []seat.Shared {
|
||||
if len(s.Shared) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]seat.Shared, 0, len(s.Shared))
|
||||
for _, x := range s.Shared {
|
||||
out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
@@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
|
||||
t.Errorf("show with the fill unread: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared
|
||||
// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step,
|
||||
// the run it waits on (§731; A-469r, A-469v).
|
||||
func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit")
|
||||
started := map[string]seat.StepResult{}
|
||||
for _, e := range f.said {
|
||||
started[e.Result.Artifact+" "+e.Result.Step] = e.Result
|
||||
}
|
||||
for _, s := range []seat.StepResult{asker, reader} {
|
||||
got, ok := started[s.Artifact+" unit"]
|
||||
if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) {
|
||||
t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok)
|
||||
}
|
||||
}
|
||||
if _, ok := started[" composition"]; !ok {
|
||||
t.Error("the composition step's step-started is not said")
|
||||
}
|
||||
pipes := memKept{}
|
||||
put(t, pipes, p.Pipeline, f.pipeline(p))
|
||||
a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waits := map[string]seat.Wait{}
|
||||
for _, w := range a.Stands {
|
||||
waits[w.Artifact+" "+w.Step] = w
|
||||
}
|
||||
want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}}
|
||||
if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] {
|
||||
t.Errorf("the asking step stands as %+v, want its ask and its run", w)
|
||||
}
|
||||
if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] {
|
||||
t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w)
|
||||
}
|
||||
if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 {
|
||||
t.Errorf("the composition step stands as %+v, want its own ask and no run", w)
|
||||
}
|
||||
// Resume asks the steps again and says their step-started again (A-469a).
|
||||
n := len(f.said)
|
||||
if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.said) == n {
|
||||
t.Error("resume said no step-started")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
|
||||
@@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) {
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose
|
||||
// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none.
|
||||
func ClaimantOf(halves []Half, seatName string) (string, bool) {
|
||||
found := ""
|
||||
for _, h := range halves {
|
||||
var named []string
|
||||
for _, e := range h.Entries {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == seatName {
|
||||
named = append(named, e.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(named) == 0:
|
||||
continue
|
||||
case len(named) > 1, found != "" && found != named[0]:
|
||||
return "", false
|
||||
}
|
||||
found = named[0]
|
||||
}
|
||||
return found, found != ""
|
||||
}
|
||||
|
||||
@@ -229,9 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
|
||||
}
|
||||
p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
if s.Registration != nil {
|
||||
if reg, ok := s.Registration(p.Module); ok {
|
||||
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
|
||||
asks, ended = s.holderless(&p, asks, now)
|
||||
} else {
|
||||
s.logf("%s has no registration yet: its pipeline is placed with no step", key)
|
||||
}
|
||||
@@ -246,9 +248,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
|
||||
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
||||
if err := s.askSteps(key, p.Commit, asks); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) {
|
||||
if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil {
|
||||
t.Error("an entry the members do not change got a pipeline")
|
||||
}
|
||||
if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 {
|
||||
// Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a).
|
||||
started, steps := 0, 0
|
||||
for _, x := range said {
|
||||
started += strings.Count(x, seat.EventPipelineStarted+".")
|
||||
steps += strings.Count(x, seat.EventStepStarted+".")
|
||||
}
|
||||
if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
|
||||
t.Errorf("placing said %v and asked %d steps", said, len(asks.ids))
|
||||
}
|
||||
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
|
||||
|
||||
@@ -0,0 +1,488 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs
|
||||
// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of
|
||||
// an ask never taken.
|
||||
type RunSettings struct {
|
||||
CapBytes int64
|
||||
Kept time.Duration
|
||||
StepBound time.Duration
|
||||
Untaken time.Duration
|
||||
}
|
||||
|
||||
// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours.
|
||||
var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
|
||||
|
||||
// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.<run key>`,
|
||||
// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.<run key>`,
|
||||
// written at the run's start.
|
||||
const (
|
||||
BoundPrefix = "bound."
|
||||
UntakenPrefix = "untaken."
|
||||
)
|
||||
|
||||
// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives.
|
||||
type RunObjects interface {
|
||||
Put(name string, data []byte) error
|
||||
// Get reads an object; ErrNoRun when it is gone.
|
||||
Get(name string) ([]byte, error)
|
||||
Delete(name string) error
|
||||
}
|
||||
|
||||
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the
|
||||
// run it bounds (A-470n).
|
||||
type RunClock interface {
|
||||
// Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision
|
||||
// (retried when another write came between); it answers the revision written.
|
||||
Bound(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// Bounded reads key: the ask it holds and its revision; ok false when it is not written.
|
||||
Bounded(key string) (ask string, revision uint64, ok bool, err error)
|
||||
// Unbound deletes key; a key gone is no fault.
|
||||
Unbound(key string) error
|
||||
// UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault.
|
||||
UnboundAt(key string, revision uint64) error
|
||||
// Has says whether key is written.
|
||||
Has(key string) (bool, error)
|
||||
// Keys are test-runs' keys.
|
||||
Keys() ([]string, error)
|
||||
}
|
||||
|
||||
// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound.
|
||||
func ParseRunBound(key string) (run string, untaken, ok bool) {
|
||||
if run, ok = strings.CutPrefix(key, BoundPrefix); ok {
|
||||
return run, false, run != ""
|
||||
}
|
||||
if run, ok = strings.CutPrefix(key, UntakenPrefix); ok {
|
||||
return run, true, run != ""
|
||||
}
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
// IsRunBound says whether a key of test-runs is a bound key rather than a run.
|
||||
func IsRunBound(key string) bool {
|
||||
_, _, ok := ParseRunBound(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines.
|
||||
func (h *Hearer) runOfAsk(ask string) (string, bool) {
|
||||
if h.TestRuns == nil || ask == "" {
|
||||
return "", false
|
||||
}
|
||||
pkeys, err := h.Store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 {
|
||||
run := sharedRun(p.Steps[i])
|
||||
return run, run != ""
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken
|
||||
// bound deleted and its clock begun, `bound.<run key>` written with test-step-bound as its time to live. An ask of no test
|
||||
// run takes nothing.
|
||||
func (h *Hearer) RunTaken(ask, node string) error {
|
||||
run, ok := h.runOfAsk(ask)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
if r.Ask != ask || r.Ended != nil {
|
||||
h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if r.Node != node {
|
||||
r.Node = node
|
||||
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("the test run %s is not written: %w", run, err)
|
||||
}
|
||||
}
|
||||
h.runStarted(run, r)
|
||||
break
|
||||
}
|
||||
if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil {
|
||||
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
|
||||
}
|
||||
// The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept.
|
||||
held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run)
|
||||
if err != nil || !ok || held != ask {
|
||||
return err
|
||||
}
|
||||
return h.Clock.UnboundAt(UntakenPrefix+run, rev)
|
||||
}
|
||||
|
||||
// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run
|
||||
// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no
|
||||
// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept.
|
||||
func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error {
|
||||
rev, err := clock.Bound(key, ask, ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, _, err := runs.Get(run)
|
||||
if err != nil && !errors.Is(err, ErrNoRun) {
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask {
|
||||
return clock.UnboundAt(key, rev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that
|
||||
// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing.
|
||||
func (h *Hearer) RunProgress(ask string) error {
|
||||
run, ok := h.runOfAsk(ask)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) {
|
||||
h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound)
|
||||
}
|
||||
|
||||
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
|
||||
// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask
|
||||
// was taken, and a bound of a run ended already, end nothing.
|
||||
func (h *Hearer) RunExpired(key string) error {
|
||||
run, untaken, ok := ParseRunBound(key)
|
||||
if !ok || h.TestRuns == nil {
|
||||
return nil
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
// An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing.
|
||||
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
|
||||
return nil
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again,
|
||||
// so an expiry heard after the bound was armed again for the run's own ask ends nothing.
|
||||
if _, _, armed, err := h.Clock.Bounded(key); err != nil {
|
||||
return fmt.Errorf("the bound %s is not read again: %w", key, err)
|
||||
} else if armed {
|
||||
h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key)
|
||||
return nil
|
||||
}
|
||||
s := h.runSettings()
|
||||
why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound)
|
||||
if untaken {
|
||||
why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken)
|
||||
}
|
||||
if h.DropRun != nil {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err)
|
||||
}
|
||||
}
|
||||
return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
|
||||
}
|
||||
|
||||
// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an
|
||||
// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It
|
||||
// says how many it ended.
|
||||
func (h *Hearer) TakeUp() (int, error) {
|
||||
if h.TestRuns == nil || h.Clock == nil {
|
||||
return 0, nil
|
||||
}
|
||||
all, err := h.Clock.Keys()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the test runs are not listed: %w", err)
|
||||
}
|
||||
ended := 0
|
||||
var first error
|
||||
for _, run := range all {
|
||||
if IsRunBound(run) {
|
||||
continue
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if err != nil || r.Ended != nil {
|
||||
continue
|
||||
}
|
||||
bound := UntakenPrefix + run
|
||||
if r.Node != "" {
|
||||
bound = BoundPrefix + run
|
||||
}
|
||||
has, err := h.Clock.Has(bound)
|
||||
if err != nil || has {
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := h.RunExpired(bound); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
ended++
|
||||
}
|
||||
return ended, first
|
||||
}
|
||||
|
||||
func (h *Hearer) runSettings() RunSettings {
|
||||
s := h.Runs
|
||||
d := DefaultRunSettings
|
||||
if s.CapBytes <= 0 {
|
||||
s.CapBytes = d.CapBytes
|
||||
}
|
||||
if s.Kept <= 0 {
|
||||
s.Kept = d.Kept
|
||||
}
|
||||
if s.StepBound <= 0 {
|
||||
s.StepBound = d.StepBound
|
||||
}
|
||||
if s.Untaken <= 0 {
|
||||
s.Untaken = d.Untaken
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set,
|
||||
// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not
|
||||
// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not
|
||||
// ended by a judgement of the one before.
|
||||
func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error {
|
||||
var r Run
|
||||
for {
|
||||
got, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests
|
||||
if end.Node != "" {
|
||||
got.Node = end.Node
|
||||
}
|
||||
if end.Output != "" {
|
||||
got.Output = end.Output
|
||||
}
|
||||
if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("the test run %s is not written: %w", run, err)
|
||||
}
|
||||
r = got
|
||||
break
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
|
||||
if err := h.Clock.Unbound(k); err != nil {
|
||||
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
var first error
|
||||
for _, reader := range r.Readers {
|
||||
if skip != nil && reader.Of() == *skip {
|
||||
continue
|
||||
}
|
||||
if err := h.endReader(reader, run, r); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming
|
||||
// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d,
|
||||
// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming
|
||||
// the new ask, its pipeline reopened (§727).
|
||||
func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
return nil // a pipeline gone has no reader left to end
|
||||
}
|
||||
i, moved := -1, false
|
||||
for j, s := range p.Steps {
|
||||
if s.Artifact != reader.Artifact || sharedRun(s) != run {
|
||||
continue
|
||||
}
|
||||
if open(s) {
|
||||
i, moved = j, false
|
||||
} else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil {
|
||||
i, moved = j, true
|
||||
}
|
||||
}
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
if moved {
|
||||
s.Rerun, s.By = true, reader.By
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
}
|
||||
}
|
||||
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil
|
||||
if r.Tests != nil {
|
||||
t := *r.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now))
|
||||
asked, err := askApproval(reader.Pipeline, &p, now, h.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended := endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
h.appendUnlogged(reader.Pipeline)
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
|
||||
// candidate starts a new run.
|
||||
const NoReaderLeft = "no reader is left"
|
||||
|
||||
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
|
||||
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
|
||||
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
|
||||
// nothing.
|
||||
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil {
|
||||
return
|
||||
}
|
||||
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
|
||||
for {
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
|
||||
return
|
||||
}
|
||||
left := r.Readers[:0:0]
|
||||
for _, o := range r.Readers {
|
||||
if o.Of() != reader {
|
||||
left = append(left, o)
|
||||
}
|
||||
}
|
||||
if len(left) == len(r.Readers) {
|
||||
return
|
||||
}
|
||||
r.Readers = left
|
||||
// The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks
|
||||
// its own, never joining a run then ended for having no reader.
|
||||
last := len(left) == 0 && r.Ended == nil
|
||||
if last {
|
||||
now := h.now()
|
||||
r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft
|
||||
}
|
||||
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
|
||||
return
|
||||
}
|
||||
if !last {
|
||||
return
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
|
||||
if err := h.Clock.Unbound(k); err != nil {
|
||||
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if h.DropRun != nil {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask
|
||||
// (A-469b).
|
||||
func open(s seat.StepResult) bool {
|
||||
return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
|
||||
}
|
||||
|
||||
// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b),
|
||||
// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732).
|
||||
func (h *Hearer) runStarted(run string, r Run) {
|
||||
for _, reader := range r.Readers {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var moved []int
|
||||
for j, s := range p.Steps {
|
||||
if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode {
|
||||
p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, ""
|
||||
moved = append(moved, j)
|
||||
}
|
||||
}
|
||||
if len(moved) == 0 {
|
||||
break
|
||||
}
|
||||
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err)
|
||||
break
|
||||
}
|
||||
for _, j := range moved {
|
||||
if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil {
|
||||
h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
@@ -73,8 +75,12 @@ type Starter struct {
|
||||
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
|
||||
// none was heard, which asks the step with none.
|
||||
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
|
||||
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
|
||||
Arm Arm
|
||||
// Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's
|
||||
// first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless.
|
||||
Holders func(seat string) ([]string, bool)
|
||||
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
|
||||
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
|
||||
Ended func(p seat.Pipeline, steps []int) error
|
||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||
Fills *Fills
|
||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||
@@ -83,6 +89,116 @@ type Starter struct {
|
||||
Store Updater
|
||||
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
|
||||
Supersede func(newer seat.Pipeline)
|
||||
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
|
||||
// (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the
|
||||
// settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends
|
||||
// the steps of a run read ended or never asked.
|
||||
TestRuns TestRuns
|
||||
Clock RunClock
|
||||
Runs RunSettings
|
||||
RunsFill func() (seat.Fill, error)
|
||||
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
|
||||
}
|
||||
|
||||
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
|
||||
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
|
||||
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
|
||||
// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat;
|
||||
// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless.
|
||||
func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) {
|
||||
if s.Holders == nil {
|
||||
return asks, nil
|
||||
}
|
||||
var none []string
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
|
||||
continue
|
||||
}
|
||||
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
|
||||
none = append(none, st.Seat)
|
||||
}
|
||||
}
|
||||
if len(none) == 0 {
|
||||
return asks, nil
|
||||
}
|
||||
sort.Strings(none)
|
||||
whys := make([]string, 0, len(none))
|
||||
for _, n := range none {
|
||||
whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n))
|
||||
}
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Ended, st.Ask, st.Shared = &now, "", nil
|
||||
if slices.Contains(none, st.Seat) {
|
||||
st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat)
|
||||
} else {
|
||||
st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ")
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now
|
||||
p.EndCount++
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why)
|
||||
if s.Ended == nil {
|
||||
return nil
|
||||
}
|
||||
return s.Ended(p, steps)
|
||||
}
|
||||
|
||||
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
|
||||
// the kept results and toolchain are carried and each ask is put.
|
||||
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
|
||||
placed := asks
|
||||
asks = s.share(key, asks)
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, s.recovered(key, commit))
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
}
|
||||
|
||||
// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a):
|
||||
// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times
|
||||
// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729).
|
||||
func (s *Starter) sayStarted(key string, asks []Asked) {
|
||||
if s.Store == nil || s.Say == nil || len(asks) == 0 {
|
||||
return
|
||||
}
|
||||
ids, runs := map[string]bool{}, map[string]bool{}
|
||||
for _, a := range asks {
|
||||
ids[a.ID] = true
|
||||
if a.Run != "" {
|
||||
runs[a.Run] = true
|
||||
}
|
||||
}
|
||||
p, _, err := s.Store.Get(key)
|
||||
if err != nil {
|
||||
s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err)
|
||||
return
|
||||
}
|
||||
at := s.now()
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
|
||||
continue
|
||||
}
|
||||
subject, ev := stepEvent(seat.EventStepStarted, p, st, at)
|
||||
if err := s.Say(subject, ev); err != nil {
|
||||
s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
|
||||
@@ -115,11 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
p := newPipeline(b, key, now)
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
if b.unheld {
|
||||
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
|
||||
} else if s.Registration != nil {
|
||||
if reg, ok := s.Registration(b.Module); ok {
|
||||
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
|
||||
asks, ended = s.holderless(&p, asks, now)
|
||||
} else {
|
||||
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
|
||||
}
|
||||
@@ -143,9 +261,11 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if s.Supersede != nil {
|
||||
s.Supersede(p)
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
|
||||
if err := s.askSteps(key, b.Commit, asks); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
@@ -187,24 +307,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
running := map[string]bool{}
|
||||
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
|
||||
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
|
||||
running, own := map[string]bool{}, map[string]bool{}
|
||||
for _, st := range p.Steps {
|
||||
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
|
||||
running[st.Ask] = true
|
||||
if open(st) && st.Ask != "" {
|
||||
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
|
||||
}
|
||||
}
|
||||
_, placed := Place(p, reg, half.Entries, now, s.Name)
|
||||
_, shared := Place(p, reg, half.Entries, now, s.Name)
|
||||
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
|
||||
var due []Asked
|
||||
for _, a := range placed {
|
||||
if running[a.ID] {
|
||||
for _, a := range shared {
|
||||
if running[a.ID] && !own[a.ID] {
|
||||
due = append(due, a)
|
||||
}
|
||||
}
|
||||
for _, a := range unshared {
|
||||
if running[a.ID] && own[a.ID] {
|
||||
a.Run = ""
|
||||
due = append(due, a)
|
||||
}
|
||||
}
|
||||
if len(due) == 0 {
|
||||
continue
|
||||
}
|
||||
s.carry(key, due)
|
||||
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
|
||||
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
asked += len(due)
|
||||
@@ -284,6 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
|
||||
}
|
||||
var asks []Asked
|
||||
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
|
||||
asks, ended := s.holderless(&p, asks, now)
|
||||
if _, err := store.Update(key, p, rev); err != nil {
|
||||
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
|
||||
continue
|
||||
@@ -299,9 +428,14 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
||||
if err := s.askSteps(key, p.Commit, asks); err != nil {
|
||||
note(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ type StepSeat struct {
|
||||
Prefix string
|
||||
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
||||
Decode func([]byte) (StepEnded, error)
|
||||
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
||||
// a seat whose ends name none.
|
||||
Shared func([]byte) []seat.Shared
|
||||
}
|
||||
|
||||
// StepSeats are the three build seats, from their contracts.
|
||||
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
|
||||
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
||||
// The seat mesh-wants answers the composition step (§22, §24).
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
||||
}
|
||||
|
||||
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
|
||||
// A-469t): the same form, under the same name.
|
||||
func wantsShared(raw []byte) []seat.Shared {
|
||||
var w wants.StepEnded
|
||||
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]seat.Shared, 0, len(w.Shared))
|
||||
for _, x := range w.Shared {
|
||||
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
|
||||
if x.AskedBy != nil {
|
||||
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
||||
@@ -118,6 +139,26 @@ type Hearer struct {
|
||||
Fills *Fills
|
||||
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
|
||||
Widens func(seat.Pipeline) []seat.Widening
|
||||
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
|
||||
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
|
||||
TestRuns TestRuns
|
||||
Clock RunClock
|
||||
Runs RunSettings
|
||||
DropRun Drop
|
||||
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
|
||||
RunOutputs RunObjects
|
||||
}
|
||||
|
||||
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
|
||||
// of a run that was never asked.
|
||||
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
|
||||
var first error
|
||||
for _, reader := range readers {
|
||||
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func (h *Hearer) now() time.Time {
|
||||
@@ -132,6 +173,13 @@ func (h *Hearer) now() time.Time {
|
||||
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
||||
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
||||
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
||||
return h.OnStepEndedShared(bs, subject, ev, nil)
|
||||
}
|
||||
|
||||
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
|
||||
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
||||
// A-469m).
|
||||
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
||||
node := strings.TrimPrefix(subject, bs.Prefix)
|
||||
if node == subject || node == "" || strings.Contains(node, ".") {
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
||||
@@ -153,7 +201,15 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
|
||||
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
|
||||
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
|
||||
case !open(p.Steps[i]):
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
return p, h.endRunOf(p, p.Steps[i], ev)
|
||||
}
|
||||
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
|
||||
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
|
||||
}
|
||||
return p, nil
|
||||
case !endOutcome(ev.Outcome):
|
||||
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
|
||||
@@ -161,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
||||
if len(shared) > 0 {
|
||||
s.Shared = shared
|
||||
}
|
||||
if ev.Tests != nil {
|
||||
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
@@ -177,6 +236,8 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
}
|
||||
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
|
||||
p.Unlogged = append(p.Unlogged, unlogged)
|
||||
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
|
||||
readers := h.sameRunReaders(&p, i, ev, now)
|
||||
from := now
|
||||
if ev.Ended != nil {
|
||||
from = ev.Ended.UTC()
|
||||
@@ -191,6 +252,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
} else if err != nil {
|
||||
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
||||
}
|
||||
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
|
||||
// finding's event the bus is slow to take) delays it past its bound (A-470o).
|
||||
runErr := h.endRunOf(p, *s, ev)
|
||||
if h.Keep != nil {
|
||||
for _, k := range ev.Kept {
|
||||
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
|
||||
@@ -209,14 +273,94 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
for _, j := range readers {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
||||
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
|
||||
h.appendUnlogged(ev.Pipeline)
|
||||
h.Ager.Age()
|
||||
h.Fills.Check()
|
||||
return p, h.said(p, asked, ended)
|
||||
if err := h.said(p, asked, ended); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, runErr
|
||||
}
|
||||
}
|
||||
|
||||
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
|
||||
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
|
||||
// answers their indexes.
|
||||
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
|
||||
asker := p.Steps[i]
|
||||
run := sharedRun(asker)
|
||||
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
|
||||
return nil
|
||||
}
|
||||
marks := map[string]RunReader{}
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil {
|
||||
for _, o := range r.Readers {
|
||||
if o.Pipeline == p.Pipeline {
|
||||
marks[o.Artifact] = o
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []int
|
||||
for j := range p.Steps {
|
||||
s := &p.Steps[j]
|
||||
if j == i || sharedRun(*s) != run {
|
||||
continue
|
||||
}
|
||||
switch m := marks[s.Artifact]; {
|
||||
case open(*s) && s.Ask == "":
|
||||
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
|
||||
s.Rerun, s.By = true, m.By
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
|
||||
if asker.Tests != nil {
|
||||
t := *asker.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
|
||||
out = append(out, j)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
|
||||
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
|
||||
// A step that asked no run ends nothing more.
|
||||
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
|
||||
return nil
|
||||
}
|
||||
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
|
||||
if ev.Tests != nil {
|
||||
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
if h.RunOutputs != nil {
|
||||
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
|
||||
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
|
||||
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
|
||||
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err == nil {
|
||||
err = h.RunOutputs.Put(run, raw)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
|
||||
}
|
||||
end.Output = run
|
||||
}
|
||||
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
|
||||
}
|
||||
|
||||
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
|
||||
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
|
||||
if asked {
|
||||
@@ -300,8 +444,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
at := h.now()
|
||||
switch event {
|
||||
case seat.EventStepEnded, seat.EventStepStarted:
|
||||
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
||||
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
|
||||
_, ev := stepEvent(event, p, *step, at)
|
||||
return h.Say(subject, ev)
|
||||
case seat.EventPipelineEnded:
|
||||
e := pipelineEnded(p)
|
||||
if h.Widens != nil {
|
||||
@@ -314,17 +458,46 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
}
|
||||
}
|
||||
|
||||
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with
|
||||
// its shared when it reads a run (§729; A-468e, A-469q).
|
||||
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
|
||||
last := p.Commit
|
||||
if p.Group != "" {
|
||||
last = p.Group
|
||||
}
|
||||
if event == seat.EventStepStarted {
|
||||
// A step-started names the run and its ask, never the step that asked it (A-469q).
|
||||
var shared []seat.Shared
|
||||
for _, x := range step.Shared {
|
||||
x.AskedBy = nil
|
||||
shared = append(shared, x)
|
||||
}
|
||||
step.Shared = shared
|
||||
}
|
||||
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
|
||||
return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
||||
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step}
|
||||
}
|
||||
|
||||
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
|
||||
const (
|
||||
budgetUnit = 500
|
||||
budgetIntegration = 50
|
||||
)
|
||||
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
|
||||
// steps read it (§726, A-466i); nil when no step counted any.
|
||||
func testCounts(steps []seat.StepResult) *seat.TestCounts {
|
||||
var t seat.TestCounts
|
||||
counted := false
|
||||
runs := map[string]bool{}
|
||||
for _, s := range steps {
|
||||
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
|
||||
if runs[run] {
|
||||
continue
|
||||
}
|
||||
runs[run] = true
|
||||
}
|
||||
if s.Tests != nil {
|
||||
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
|
||||
}
|
||||
@@ -380,7 +553,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
||||
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
|
||||
// pipeline's end (A-104).
|
||||
continue
|
||||
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
|
||||
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
|
||||
return false
|
||||
}
|
||||
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
||||
@@ -401,3 +574,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
|
||||
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
|
||||
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
var first error
|
||||
for _, i := range steps {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
|
||||
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(p.Pipeline)
|
||||
if len(steps) > 0 {
|
||||
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
|
||||
}
|
||||
if err := h.said(p, false, true); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
@@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline
|
||||
// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing.
|
||||
func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) {
|
||||
h, store, said, _, _, _ := fixture()
|
||||
e := store[pkey]
|
||||
e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder
|
||||
e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked
|
||||
store[pkey] = e
|
||||
if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatalf("an ask no step names ended something: %v", err)
|
||||
}
|
||||
if err := h.NotTaken("a1", AskBound); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := store[pkey].p
|
||||
if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) ||
|
||||
p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) {
|
||||
t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat)
|
||||
}
|
||||
// Its end taken once: the bound's expiry heard again changes nothing.
|
||||
n := len(*said)
|
||||
if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n {
|
||||
t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is
|
||||
// not armed is not put (§535, §647).
|
||||
func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) {
|
||||
var order []string
|
||||
asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil })
|
||||
arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil })
|
||||
asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}}
|
||||
if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" {
|
||||
t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err)
|
||||
}
|
||||
order = nil
|
||||
failing := Arm(func(string) error { return errors.New("bucket gone") })
|
||||
if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 {
|
||||
t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err)
|
||||
}
|
||||
}
|
||||
|
||||
type askerFunc func(accept, id string, body any) error
|
||||
|
||||
func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) }
|
||||
@@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
break
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node {
|
||||
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
|
||||
break
|
||||
}
|
||||
p.Steps[i].Node = node
|
||||
@@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
// whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started
|
||||
// (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended
|
||||
// `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its
|
||||
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through
|
||||
// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
|
||||
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves
|
||||
// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
|
||||
// request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended.
|
||||
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) {
|
||||
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) {
|
||||
if newer.Run != "" || newer.Source <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri
|
||||
if key == newer.Pipeline {
|
||||
continue
|
||||
}
|
||||
done, err := h.supersedeOne(key, newer, why, drop, unbound)
|
||||
done, err := h.supersedeOne(key, newer, why, drop)
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
@@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string {
|
||||
return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module)
|
||||
}
|
||||
|
||||
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) {
|
||||
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil || !superseded(p, newer) {
|
||||
@@ -110,10 +110,11 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
var changed []int
|
||||
for i := range p.Steps {
|
||||
s := &p.Steps[i]
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if s.Ask != "" && BoundSeat(s.Seat) {
|
||||
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
|
||||
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
|
||||
drops = append(drops, *s)
|
||||
}
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
|
||||
@@ -136,17 +137,15 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
if len(changed) > 0 {
|
||||
h.appendUnlogged(key)
|
||||
}
|
||||
for _, i := range changed {
|
||||
h.leaveRun(key, p.Steps[i], why)
|
||||
}
|
||||
for _, s := range drops {
|
||||
if drop != nil {
|
||||
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if unbound != nil {
|
||||
if err := unbound(s.Ask); err != nil {
|
||||
h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := h.said(p, false, true); err != nil && first == nil {
|
||||
first = err
|
||||
|
||||
@@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) {
|
||||
}
|
||||
type dropped struct{ seat, node, ask string }
|
||||
var drops []dropped
|
||||
var unbound []string
|
||||
drop := func(s, node, ask, why string) error {
|
||||
if !strings.Contains(why, "c2") {
|
||||
t.Errorf("the drop of %s names no newer commit: %q", ask, why)
|
||||
@@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) {
|
||||
drops = append(drops, dropped{s, node, ask})
|
||||
return nil
|
||||
}
|
||||
unbind := func(ask string) error { unbound = append(unbound, ask); return nil }
|
||||
newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline {
|
||||
return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub,
|
||||
Outcome: seat.PipelineOutcomeRunning}
|
||||
@@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) {
|
||||
{"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)},
|
||||
{"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)},
|
||||
} {
|
||||
if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil {
|
||||
if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil {
|
||||
t.Errorf("%s superseded %d (%v)", c.name, n, err)
|
||||
}
|
||||
}
|
||||
*said = nil
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil {
|
||||
t.Fatalf("a changed fingerprint superseded %d (%v)", n, err)
|
||||
}
|
||||
p, _, _ := store.Get(pkey)
|
||||
@@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) {
|
||||
}
|
||||
}
|
||||
want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}}
|
||||
if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) {
|
||||
t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want)
|
||||
if !slices.Equal(drops, want) {
|
||||
t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want)
|
||||
}
|
||||
if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" {
|
||||
t.Errorf("said %v, want four step-ended then pipeline-ended", *said)
|
||||
}
|
||||
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 {
|
||||
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 {
|
||||
t.Errorf("a superseded pipeline was superseded again: %d", n)
|
||||
}
|
||||
|
||||
@@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) {
|
||||
pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa",
|
||||
Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}}
|
||||
h.Store = pub
|
||||
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 {
|
||||
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 {
|
||||
t.Errorf("a publish of the same version superseded %d", n)
|
||||
}
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil {
|
||||
t.Fatalf("a newer publish superseded %d (%v)", n, err)
|
||||
}
|
||||
if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") {
|
||||
@@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) {
|
||||
if _, _, ok := ParseDropBound("step-ab"); ok {
|
||||
t.Error("an ask's bound was read as a drop's")
|
||||
}
|
||||
if IsAskKey(DropBoundKey("s", "step-ab")) {
|
||||
t.Error("a drop's bound was taken for an ask's")
|
||||
}
|
||||
err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why")
|
||||
if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 {
|
||||
t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered)
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested
|
||||
// packages, written create-only by the step that asks the run and changed by compare-and-set.
|
||||
type TestRuns interface {
|
||||
// Create writes key when it is absent; ErrExists when it is written already.
|
||||
Create(key string, r Run) error
|
||||
// Get reads key and its revision; ErrNoRun when it is absent.
|
||||
Get(key string) (Run, uint64, error)
|
||||
// Update writes key at the revision read; ErrChanged when another write came between.
|
||||
Update(key string, r Run, revision uint64) (uint64, error)
|
||||
// Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No
|
||||
// write of a run puts a value judged from an earlier read over a later one (the kind of the stale write).
|
||||
Delete(key string, revision uint64) error
|
||||
}
|
||||
|
||||
// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by
|
||||
// whom (A-470d), so the marks are in the key and a restart takes them up.
|
||||
type Run struct {
|
||||
seat.TestRun
|
||||
Readers []RunReader `json:"readers,omitempty"`
|
||||
}
|
||||
|
||||
// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun
|
||||
// whose new run moves it (A-470d).
|
||||
type RunReader struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Artifact string `json:"artifact"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// Of is the reader as the seat names it.
|
||||
func (r RunReader) Of() seat.TestRunReader {
|
||||
return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact}
|
||||
}
|
||||
|
||||
// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact.
|
||||
func (r Run) Answer() seat.TestRun {
|
||||
t := r.TestRun
|
||||
t.Readers = nil
|
||||
for _, o := range r.Readers {
|
||||
t.Readers = append(t.Readers, o.Of())
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// ErrNoRun is a Get of a test-runs key that is not written.
|
||||
var ErrNoRun = errors.New("no test run under the key")
|
||||
|
||||
// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-<id>` for a run on request, else `<branch>.<revision>`
|
||||
// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A
|
||||
// pipeline naming neither has no candidate, and its steps share no run.
|
||||
func CandidateOf(p seat.Pipeline) (string, bool) {
|
||||
var key string
|
||||
var err error
|
||||
if p.Run != "" {
|
||||
key, err = nec.RunKey(p.Run)
|
||||
} else {
|
||||
key, err = nec.RevisionKey(p.Branch, p.Revision)
|
||||
}
|
||||
return key, err == nil
|
||||
}
|
||||
|
||||
// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint,
|
||||
// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner
|
||||
// holds, so two artifacts with one fetch meet and two with different fetches do not.
|
||||
func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string {
|
||||
paths := append([]string(nil), fetch.Paths...)
|
||||
sort.Strings(paths)
|
||||
h := sha256.New()
|
||||
for _, part := range []string{fingerprint, level, status, fetch.Dir} {
|
||||
h.Write([]byte(part + "\n"))
|
||||
}
|
||||
h.Write([]byte(strings.Join(paths, "\n")))
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// TestRunKey is a test run's key in test-runs (§725): `<candidate key>.<level>.<digest>`, the level and the digest each
|
||||
// escaped as a key's name, the candidate already in its key form.
|
||||
func TestRunKey(candidate, level, digest string) string {
|
||||
return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest)
|
||||
}
|
||||
|
||||
// sharedRun is the one run a step's Shared names, empty when it names none.
|
||||
func sharedRun(s seat.StepResult) string {
|
||||
if len(s.Shared) == 0 {
|
||||
return ""
|
||||
}
|
||||
return s.Shared[0].Run
|
||||
}
|
||||
|
||||
// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask
|
||||
// of its own (A-469m).
|
||||
func readsRun(s seat.StepResult, run string) bool {
|
||||
return run != "" && sharedRun(s) == run && s.Ask == ""
|
||||
}
|
||||
|
||||
// addReaders adds each reader not in r's readers yet, in order.
|
||||
func addReaders(r *Run, readers []seat.TestRunReader) {
|
||||
for _, n := range readers {
|
||||
have := false
|
||||
for _, o := range r.Readers {
|
||||
have = have || o.Of() == n
|
||||
}
|
||||
if !have {
|
||||
r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is.
|
||||
func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
|
||||
if store == nil || ask == "" {
|
||||
return nil, nil
|
||||
}
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 {
|
||||
return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run;
|
||||
// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their
|
||||
// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read:
|
||||
// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again,
|
||||
// kept once by its id. It answers the asks still to put.
|
||||
func (s *Starter) share(key string, asks []Asked) []Asked {
|
||||
if s.TestRuns == nil {
|
||||
return asks
|
||||
}
|
||||
var out []Asked
|
||||
joined := map[string]Run{}
|
||||
for _, a := range asks {
|
||||
if a.Run == "" {
|
||||
out = append(out, a)
|
||||
continue
|
||||
}
|
||||
run, ok, err := s.claimRun(a)
|
||||
var full *fullErr
|
||||
switch {
|
||||
case errors.As(err, &full):
|
||||
// At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d).
|
||||
s.logf("the test run %s of %s is not asked: %v", a.Run, key, err)
|
||||
if s.EndReaders != nil {
|
||||
if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil {
|
||||
s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err)
|
||||
}
|
||||
}
|
||||
case err != nil:
|
||||
// A run that cannot be written or read is asked as the step's own, said: slower, never lost.
|
||||
s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err)
|
||||
out = append(out, a)
|
||||
case ok:
|
||||
out = append(out, a)
|
||||
default:
|
||||
joined[a.Run] = run
|
||||
}
|
||||
}
|
||||
if len(joined) > 0 {
|
||||
s.readOthers(key, joined)
|
||||
// A run that ended within the candidate is read at once (§724): its readers end with its result.
|
||||
for run, r := range joined {
|
||||
if r.Ended == nil || s.EndReaders == nil {
|
||||
continue
|
||||
}
|
||||
var readers []seat.TestRunReader
|
||||
for _, a := range asks {
|
||||
if a.Run == run {
|
||||
readers = a.Readers
|
||||
}
|
||||
}
|
||||
if err := s.EndReaders(run, readers, r.TestRun); err != nil {
|
||||
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill.
|
||||
type fullErr struct{ fill seat.Fill }
|
||||
|
||||
func (e *fullErr) Error() string {
|
||||
return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap)
|
||||
}
|
||||
|
||||
// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i),
|
||||
// never the manifest's ceiling.
|
||||
func (s *Starter) TestRunsFill() (seat.Fill, error) {
|
||||
if s.RunsFill == nil {
|
||||
return seat.Fill{}, errors.New("the runner reads no fill of test-runs")
|
||||
}
|
||||
f, err := s.RunsFill()
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
f.Cap = s.Runs.CapBytes
|
||||
if f.Cap <= 0 {
|
||||
f.Cap = DefaultRunSettings.CapBytes
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said).
|
||||
func (s *Starter) atCap() error {
|
||||
if s.RunsFill == nil {
|
||||
return nil
|
||||
}
|
||||
f, err := s.TestRunsFill()
|
||||
if err != nil {
|
||||
s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err)
|
||||
return nil
|
||||
}
|
||||
if f.Bytes >= f.Cap {
|
||||
return &fullErr{fill: f}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// started writes a new run and its untaken bound (§725, A-470).
|
||||
func (s *Starter) started(a Asked, write func() error) error {
|
||||
if err := s.atCap(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := write(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Clock != nil {
|
||||
untaken := s.Runs.Untaken
|
||||
if untaken <= 0 {
|
||||
untaken = DefaultRunSettings.Untaken
|
||||
}
|
||||
if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil {
|
||||
s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads.
|
||||
func (s *Starter) claimRun(a Asked) (Run, bool, error) {
|
||||
now := s.now()
|
||||
value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}}
|
||||
addReaders(&value, a.Readers)
|
||||
for {
|
||||
err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) })
|
||||
if err == nil {
|
||||
return value, true, nil
|
||||
}
|
||||
var full *fullErr
|
||||
if errors.As(err, &full) {
|
||||
return Run{}, false, err
|
||||
}
|
||||
if !errors.Is(err, ErrExists) {
|
||||
return Run{}, false, err
|
||||
}
|
||||
r, rev, err := s.TestRuns.Get(a.Run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
continue // deleted meanwhile: written again
|
||||
}
|
||||
if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
if r.Ask == a.ID {
|
||||
return r, true, nil
|
||||
}
|
||||
if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun {
|
||||
err := s.started(a, func() error {
|
||||
_, err := s.TestRuns.Update(a.Run, value, rev)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
return value, true, nil
|
||||
}
|
||||
addReaders(&r, a.Readers)
|
||||
if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
return r, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker
|
||||
// in Shared (A-469m, A-469q).
|
||||
func (s *Starter) readOthers(key string, joined map[string]Run) {
|
||||
if s.Store == nil {
|
||||
return
|
||||
}
|
||||
by := map[string]*seat.SharedAskedBy{}
|
||||
for run, r := range joined {
|
||||
asker, err := askerOf(s.Store, r.Ask)
|
||||
if err != nil {
|
||||
s.logf("the step that asked the test run %s is not read: %v", run, err)
|
||||
}
|
||||
by[run] = asker
|
||||
}
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
if err != nil {
|
||||
s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
r, ok := joined[sharedRun(*st)]
|
||||
if !ok || !open(*st) {
|
||||
continue
|
||||
}
|
||||
if r.Node != "" {
|
||||
st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b)
|
||||
}
|
||||
st.Ask = ""
|
||||
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
|
||||
}
|
||||
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// joinRun adds reader to run's readers by compare-and-set.
|
||||
func (s *Starter) joinRun(run string, reader seat.TestRunReader) {
|
||||
if s.TestRuns == nil {
|
||||
return
|
||||
}
|
||||
for {
|
||||
r, rev, err := s.TestRuns.Get(run)
|
||||
if err != nil {
|
||||
s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err)
|
||||
return
|
||||
}
|
||||
addReaders(&r, []seat.TestRunReader{reader})
|
||||
if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the
|
||||
// passing ones it kept and the carried ones it skipped, and the build seat's output.
|
||||
type RunOutput struct {
|
||||
Packages []seat.TestRunPackage `json:"packages"`
|
||||
Output string `json:"output,omitempty"`
|
||||
}
|
||||
|
||||
// Gone is what a run whose output object is gone answers in place of its packages (§725, §733).
|
||||
const Gone = "the run's output is gone"
|
||||
|
||||
// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object
|
||||
// store: each run with its packages read from its output object, and the state's fill.
|
||||
type RunsRead struct {
|
||||
Runs Kept
|
||||
// Object reads an object of test-runs-objects; ErrNoRun when it is gone.
|
||||
Object func(name string) ([]byte, error)
|
||||
Fill func() (seat.Fill, error)
|
||||
}
|
||||
|
||||
// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one.
|
||||
func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) {
|
||||
all, err := r.Runs.Keys()
|
||||
if err != nil {
|
||||
return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err)
|
||||
}
|
||||
sort.Strings(all)
|
||||
a := seat.TestRunsAnswer{Runs: []seat.TestRun{}}
|
||||
for _, key := range all {
|
||||
if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.Runs.Value(key)
|
||||
if err != nil || raw == nil {
|
||||
continue
|
||||
}
|
||||
var run Run
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
continue
|
||||
}
|
||||
if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) {
|
||||
continue
|
||||
}
|
||||
t := run.Answer()
|
||||
t.Key = key
|
||||
r.packages(&t)
|
||||
a.Runs = append(a.Runs, t)
|
||||
}
|
||||
if r.Fill != nil {
|
||||
f, err := r.Fill()
|
||||
if err != nil {
|
||||
return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err)
|
||||
}
|
||||
a.Fill = f
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// packages names t's packages from its output object, or says it gone (§725, A-466v).
|
||||
func (r RunsRead) packages(t *seat.TestRun) {
|
||||
if t.Output == "" || r.Object == nil {
|
||||
return
|
||||
}
|
||||
data, err := r.Object(t.Output)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
t.Gone = Gone
|
||||
return
|
||||
}
|
||||
var out RunOutput
|
||||
if err == nil {
|
||||
err = json.Unmarshal(data, &out)
|
||||
}
|
||||
if err != nil {
|
||||
t.Gone = "the run's output is not read: " + err.Error()
|
||||
return
|
||||
}
|
||||
t.Packages = out.Packages
|
||||
}
|
||||
|
||||
// readsModule says whether a reader of run is a step of module, its artifact `<module>:<name>`.
|
||||
func readsModule(run Run, module string) bool {
|
||||
for _, o := range run.Readers {
|
||||
if strings.HasPrefix(o.Artifact, module+":") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the
|
||||
// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output
|
||||
// object and its key deleted. It answers the run deleted and the readers it ended.
|
||||
func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) {
|
||||
h := v.Hearer
|
||||
if h.TestRuns == nil {
|
||||
return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs")
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(in.Key)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key)
|
||||
}
|
||||
if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err)
|
||||
}
|
||||
if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, err
|
||||
}
|
||||
why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why)
|
||||
var ended []seat.TestRunReader
|
||||
if r.Ended == nil {
|
||||
if h.DropRun != nil && r.Ask != "" {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err)
|
||||
}
|
||||
}
|
||||
if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, err
|
||||
}
|
||||
for _, o := range r.Readers {
|
||||
ended = append(ended, o.Of())
|
||||
}
|
||||
}
|
||||
if h.RunOutputs != nil {
|
||||
// The run's object is named by its key (endRunOf); one the run named otherwise is deleted too.
|
||||
for _, name := range []string{in.Key, r.Output} {
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if err := h.RunOutputs.Delete(name); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it.
|
||||
for {
|
||||
now, rev, err := h.TestRuns.Get(in.Key)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err)
|
||||
}
|
||||
if now.Ask != r.Ask {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask)
|
||||
}
|
||||
if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
|
||||
_ = h.Clock.Unbound(k)
|
||||
}
|
||||
}
|
||||
gone := r.Answer()
|
||||
gone.Key = in.Key
|
||||
now := h.now()
|
||||
if gone.Ended == nil {
|
||||
gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why
|
||||
}
|
||||
return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds
|
||||
// and its revision.
|
||||
type clockMem struct {
|
||||
runs testRunsMem
|
||||
bound map[string]time.Duration
|
||||
asks map[string]string
|
||||
revs map[string]uint64
|
||||
rev uint64
|
||||
// before is called once at the next Bound, a write that comes between its caller's read and its write.
|
||||
before func()
|
||||
}
|
||||
|
||||
func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b := c.before; b != nil {
|
||||
c.before = nil
|
||||
b()
|
||||
}
|
||||
if c.asks == nil {
|
||||
c.asks, c.revs = map[string]string{}, map[string]uint64{}
|
||||
}
|
||||
c.rev++
|
||||
c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev
|
||||
return c.rev, nil
|
||||
}
|
||||
func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
|
||||
_, ok := c.bound[key]
|
||||
return c.asks[key], c.revs[key], ok, nil
|
||||
}
|
||||
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
|
||||
func (c *clockMem) UnboundAt(key string, rev uint64) error {
|
||||
if c.revs[key] == rev {
|
||||
delete(c.bound, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
|
||||
func (c *clockMem) Keys() ([]string, error) {
|
||||
var out []string
|
||||
for k := range c.runs {
|
||||
out = append(out, k)
|
||||
}
|
||||
for k := range c.bound {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type objectsMem map[string][]byte
|
||||
|
||||
func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil }
|
||||
func (o objectsMem) Get(name string) ([]byte, error) {
|
||||
if d, ok := o[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, ErrNoRun
|
||||
}
|
||||
func (o objectsMem) Delete(name string) error { delete(o, name); return nil }
|
||||
|
||||
// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing
|
||||
// one directory.
|
||||
type runFixture struct {
|
||||
store memStore
|
||||
runs testRunsMem
|
||||
clock *clockMem
|
||||
objects objectsMem
|
||||
asks *stepAsks
|
||||
drops []string
|
||||
said []seat.StepEvent
|
||||
logged []string
|
||||
fill int64
|
||||
s *Starter
|
||||
h *Hearer
|
||||
}
|
||||
|
||||
func newRunFixture() *runFixture {
|
||||
f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}}
|
||||
f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}}
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
|
||||
say := func(_ string, body any) error {
|
||||
if e, ok := body.(seat.StepEvent); ok {
|
||||
f.said = append(f.said, e)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects,
|
||||
Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) },
|
||||
DropRun: func(seatName, node, ask, why string) error {
|
||||
f.drops = append(f.drops, seatName+" "+node+" "+ask)
|
||||
return nil
|
||||
}}
|
||||
f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks,
|
||||
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders,
|
||||
RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
|
||||
t.Helper()
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := f.store.Get(Key("main", source, "app"))
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline {
|
||||
q, _, _ := f.store.Get(p.Pipeline)
|
||||
return q
|
||||
}
|
||||
|
||||
// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken
|
||||
// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and
|
||||
// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run
|
||||
// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended
|
||||
// run is aged with its output object in the step log's order.
|
||||
func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
t.Run("clock", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
|
||||
f.s.Runs = f.h.Runs
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 {
|
||||
t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound)
|
||||
}
|
||||
// A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again.
|
||||
if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode {
|
||||
t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome)
|
||||
}
|
||||
n := len(f.said)
|
||||
if err := f.h.RunTaken(unit.Ask, "ace"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome)
|
||||
}
|
||||
}
|
||||
if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
|
||||
t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound)
|
||||
}
|
||||
if _, ok := f.clock.bound[UntakenPrefix+run]; ok {
|
||||
t.Error("the untaken bound of a taken run is kept")
|
||||
}
|
||||
// An untaken bound of a taken run ends nothing; progress begins the clock again.
|
||||
if err := f.h.RunExpired(UntakenPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(f.clock.bound, BoundPrefix+run)
|
||||
if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
|
||||
t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil {
|
||||
t.Fatal("a run is ended by its untaken bound after it was taken")
|
||||
}
|
||||
// An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n).
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask {
|
||||
t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run])
|
||||
}
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, _, _ := f.runs.Get(run)
|
||||
if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") {
|
||||
t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r)
|
||||
}
|
||||
if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask {
|
||||
t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops)
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said {
|
||||
t.Errorf("the %s unit step = %+v, want could not run with the run", art, s)
|
||||
}
|
||||
}
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun {
|
||||
t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome)
|
||||
}
|
||||
if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) {
|
||||
t.Errorf("the ignored step-ended is not journaled: %q", f.logged)
|
||||
}
|
||||
// An untaken run whose bound passed while the runner was away ends at the start.
|
||||
lint := stepNamed(p, "app:program", "lint").Shared[0].Run
|
||||
delete(f.clock.bound, UntakenPrefix+lint)
|
||||
if n, err := f.h.TakeUp(); n != 1 || err != nil {
|
||||
t.Fatalf("take-up ended %d, %v; want the lint run", n, err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") {
|
||||
t.Errorf("the lint run taken up = %+v", r)
|
||||
}
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20
|
||||
p := f.start(t, 4, 9)
|
||||
for _, a := range f.asks.ids {
|
||||
if a == stepNamed(p, "app:program", "unit").Ask {
|
||||
t.Error("a unit run is asked at the cap")
|
||||
}
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") {
|
||||
t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("age", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil)
|
||||
f.objects["out-1"] = []byte("ok")
|
||||
now := agedNow
|
||||
a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store},
|
||||
Aged: agedState{}, Now: func() time.Time { return now },
|
||||
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
|
||||
if from > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
|
||||
What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil
|
||||
}}
|
||||
a.Age()
|
||||
if _, _, err := f.runs.Get(run); err != nil {
|
||||
t.Fatal("a run is aged before test-runs-kept passed")
|
||||
}
|
||||
now = now.Add(2 * 24 * time.Hour)
|
||||
a.Age()
|
||||
if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 {
|
||||
t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// agedDeleter is a memStore with the delete the ager makes.
|
||||
type agedDeleter struct{ memStore }
|
||||
|
||||
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
|
||||
|
||||
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
|
||||
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
|
||||
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
|
||||
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
|
||||
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
a := f.start(t, 4, 9)
|
||||
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
|
||||
unit := stepNamed(a, "app:program", "unit")
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
|
||||
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
s := stepNamed(p, art, "unit")
|
||||
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
|
||||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
|
||||
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
|
||||
}
|
||||
}
|
||||
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
|
||||
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
|
||||
}
|
||||
}
|
||||
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) {
|
||||
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
|
||||
}
|
||||
saidReaders := 0
|
||||
for _, e := range f.said {
|
||||
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
|
||||
saidReaders++
|
||||
}
|
||||
}
|
||||
if saidReaders != 4 {
|
||||
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
|
||||
}
|
||||
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
|
||||
lint := stepNamed(a, "app:program", "lint")
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
|
||||
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
|
||||
}
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
|
||||
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
|
||||
}
|
||||
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
|
||||
c := newRunFixture()
|
||||
p := c.start(t, 4, 9)
|
||||
comp := stepNamed(p, "", "composition")
|
||||
bs := stepSeatOf(t, "mesh-wants")
|
||||
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
|
||||
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
|
||||
ev, err := bs.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
|
||||
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
|
||||
}
|
||||
}
|
||||
|
||||
// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed
|
||||
// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's
|
||||
// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the
|
||||
// pipeline says pipeline-ended again; a step a person accepted does not move.
|
||||
func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
// takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader
|
||||
// with rerun false, its step passed already, says no step-ended again (A-470l).
|
||||
for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} {
|
||||
accepted := c.accepted
|
||||
f := newRunFixture()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
|
||||
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
|
||||
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p := f.start(t, 4, 9)
|
||||
e := f.store[p.Pipeline]
|
||||
e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build
|
||||
f.store[p.Pipeline] = e
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
end := func(ask string, outcome string, rerun bool) {
|
||||
t.Helper()
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome,
|
||||
Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
end(unit.Ask, "failed", false)
|
||||
if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" {
|
||||
t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome)
|
||||
}
|
||||
if accepted {
|
||||
e := f.store[p.Pipeline]
|
||||
i := stepIndex(e.p, "unit", "app:program")
|
||||
e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"}
|
||||
f.store[p.Pipeline] = e
|
||||
}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := a.Result.Ask
|
||||
if c.takeUp {
|
||||
run := unit.Shared[0].Run
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already
|
||||
if _, err := f.runs.Update(run, r, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := f.store[p.Pipeline]
|
||||
e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed
|
||||
f.store[p.Pipeline] = e
|
||||
for k := range f.clock.bound {
|
||||
delete(f.clock.bound, k)
|
||||
}
|
||||
said := len(f.said)
|
||||
if n, err := f.h.TakeUp(); err != nil || n != 1 {
|
||||
t.Fatalf("take-up = %d, %v", n, err)
|
||||
}
|
||||
for _, ev := range f.said[said:] {
|
||||
if ev.Result.Artifact == "app:program" {
|
||||
t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev)
|
||||
}
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun {
|
||||
t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run",
|
||||
stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome)
|
||||
}
|
||||
continue
|
||||
}
|
||||
r, _, _ := f.runs.Get(unit.Shared[0].Run)
|
||||
if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") {
|
||||
t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again)
|
||||
}
|
||||
end(again, "passed", true)
|
||||
q := f.pipeline(p)
|
||||
prog := stepNamed(q, "app:program", "unit")
|
||||
if accepted {
|
||||
if prog.Outcome != seat.StepResultOutcomeAccepted {
|
||||
t.Errorf("an accepted step moved to %s", prog.Outcome)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again {
|
||||
t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog)
|
||||
}
|
||||
if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 {
|
||||
t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its
|
||||
// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the
|
||||
// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act
|
||||
// recorded it does nothing.
|
||||
func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint")
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed",
|
||||
Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipes := memKept{}
|
||||
for k, e := range f.runs {
|
||||
put(t, pipes, k, e.r)
|
||||
}
|
||||
// The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i).
|
||||
f.s.Runs.CapBytes, f.fill = 1<<20, 9
|
||||
bucket := f.s.RunsFill
|
||||
f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err }
|
||||
read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill}
|
||||
a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"})
|
||||
if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) ||
|
||||
a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 {
|
||||
t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err)
|
||||
}
|
||||
delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages
|
||||
if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 {
|
||||
t.Errorf("a run whose output is gone = %+v", a.Runs)
|
||||
}
|
||||
acts := &actLog{}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts}
|
||||
if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil {
|
||||
t.Fatal("a drop with no hand act recorded was done")
|
||||
}
|
||||
f.objects[lint.Shared[0].Run] = []byte("{}")
|
||||
d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"})
|
||||
if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 {
|
||||
t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops)
|
||||
}
|
||||
if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil {
|
||||
t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run])
|
||||
}
|
||||
if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") {
|
||||
t.Errorf("a reader of the dropped run = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called
|
||||
// once, just before that write.
|
||||
type racingRuns struct {
|
||||
testRunsMem
|
||||
beforeUpdate, beforeDelete func()
|
||||
}
|
||||
|
||||
func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) {
|
||||
if b := r.beforeUpdate; b != nil {
|
||||
r.beforeUpdate = nil
|
||||
b()
|
||||
}
|
||||
return r.testRunsMem.Update(key, v, rev)
|
||||
}
|
||||
|
||||
func (r *racingRuns) Delete(key string, rev uint64) error {
|
||||
if b := r.beforeDelete; b != nil {
|
||||
r.beforeDelete = nil
|
||||
b()
|
||||
}
|
||||
return r.testRunsMem.Delete(key, rev)
|
||||
}
|
||||
|
||||
// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design
|
||||
// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the
|
||||
// latest read, and a judgement of a run ends only the run under the ask it judged.
|
||||
func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
asked := func(f *runFixture, run string) {
|
||||
t.Helper()
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"}
|
||||
if _, err := f.runs.Update(run, r, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
race func(t *testing.T, f *runFixture, p seat.Pipeline, run string)
|
||||
}{
|
||||
{"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
_ = f.h.RunExpired(BoundPrefix + run)
|
||||
}},
|
||||
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
f.h.DropRun = func(string, string, string, string) error {
|
||||
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
|
||||
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
|
||||
return nil
|
||||
}
|
||||
for _, a := range []string{"app:program", "app:tools"} {
|
||||
f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled")
|
||||
}
|
||||
}},
|
||||
{"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil)
|
||||
runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
|
||||
a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow },
|
||||
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
|
||||
if from > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
|
||||
What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil
|
||||
}}
|
||||
a.Age()
|
||||
}},
|
||||
{"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
|
||||
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil {
|
||||
t.Error("a drop whose key holds a run asked since answers no error")
|
||||
}
|
||||
}},
|
||||
{"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) }
|
||||
_ = f.h.RunProgress(unit.Ask)
|
||||
if _, ok := f.clock.bound[BoundPrefix+run]; ok {
|
||||
t.Error("a clock written from a read before the run's end is kept")
|
||||
}
|
||||
asked(f, run) // the key's run is asked again, so the check below holds
|
||||
}},
|
||||
{"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
asked(f, run)
|
||||
_ = f.h.RunExpired(BoundPrefix + run)
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
run := stepNamed(p, "app:program", "unit").Shared[0].Run
|
||||
c.race(t, f, p, run)
|
||||
if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil {
|
||||
t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A late signal of a replaced run arms no bound and is said in the journal (A-470n).
|
||||
t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
asked(f, run)
|
||||
delete(f.clock.bound, UntakenPrefix+run)
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
_ = f.h.RunProgress(unit.Ask)
|
||||
late := 0
|
||||
for _, l := range f.logged {
|
||||
if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") {
|
||||
late++
|
||||
}
|
||||
}
|
||||
if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 {
|
||||
t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged)
|
||||
}
|
||||
})
|
||||
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
|
||||
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
|
||||
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
|
||||
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
endedFirst := false
|
||||
f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) {
|
||||
r, _, _ := f.runs.Get(run)
|
||||
endedFirst = r.Ended != nil
|
||||
return "alice", nil
|
||||
}}
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !endedFirst {
|
||||
t.Error("the step's finding was written before its run's end")
|
||||
}
|
||||
})
|
||||
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
|
||||
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
|
||||
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
|
||||
_, _ = f.runs.Update(run, r, rev)
|
||||
}}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
|
||||
if err != nil {
|
||||
t.Fatalf("a rerun whose read was changed before its write: %v", err)
|
||||
}
|
||||
r, _, _ := f.runs.Get(run)
|
||||
joined := false
|
||||
for _, o := range r.Readers {
|
||||
joined = joined || o.Pipeline == "main.9.app"
|
||||
}
|
||||
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
|
||||
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -27,8 +27,8 @@ type Verbs struct {
|
||||
Candidate func(group, branch string, revision int64) (Half, error)
|
||||
// Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded.
|
||||
Acts views.HandActs
|
||||
// Drop drops the bound of an ask a cancelled pipeline no longer waits on.
|
||||
Drop func(ask string) error
|
||||
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
|
||||
Runs RunsRead
|
||||
}
|
||||
|
||||
func (v Verbs) now() time.Time { return v.Hearer.now() }
|
||||
@@ -185,8 +185,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er
|
||||
return seat.AcceptAnswer{Result: p.Steps[i]}, nil
|
||||
}
|
||||
|
||||
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in
|
||||
// the hand-act log first (§394); it says pipeline-ended.
|
||||
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded
|
||||
// in the hand-act log first (§394, §726); it says pipeline-ended.
|
||||
func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) {
|
||||
h := v.Hearer
|
||||
recorded := false
|
||||
@@ -205,14 +205,14 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
recorded = true
|
||||
}
|
||||
now := h.now()
|
||||
var drop []string
|
||||
var leave []seat.StepResult
|
||||
for i := range p.Steps {
|
||||
s := &p.Steps[i]
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if s.Ask != "" {
|
||||
drop = append(drop, s.Ask)
|
||||
if sharedRun(*s) != "" && s.Seat != wants.Seat {
|
||||
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
|
||||
}
|
||||
s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now
|
||||
}
|
||||
@@ -223,12 +223,8 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
} else if err != nil {
|
||||
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
|
||||
}
|
||||
if v.Drop != nil {
|
||||
for _, ask := range drop {
|
||||
if err := v.Drop(ask); err != nil {
|
||||
h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err)
|
||||
}
|
||||
}
|
||||
for _, s := range leave {
|
||||
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
|
||||
}
|
||||
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
|
||||
}
|
||||
@@ -242,6 +238,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if in.Step == seat.StepResultStepApproval {
|
||||
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
|
||||
}
|
||||
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
|
||||
if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
|
||||
return v.rerunTest(caller, in, p.Steps[i])
|
||||
}
|
||||
}
|
||||
var ask *Asked
|
||||
p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error {
|
||||
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
|
||||
@@ -289,7 +290,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -301,6 +302,173 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil
|
||||
}
|
||||
|
||||
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
|
||||
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
|
||||
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
|
||||
h := v.Hearer
|
||||
run := sharedRun(st)
|
||||
if open(st) {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline)
|
||||
}
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if err != nil && !errors.Is(err, ErrNoRun) {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
exists := err == nil
|
||||
reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact}
|
||||
read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended)
|
||||
join := exists && r.Ended == nil
|
||||
var ask *Asked
|
||||
if !join && !read {
|
||||
p, _, err := h.Store.Get(in.Pipeline)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
if ask, err = v.askOf(p, st, caller, h.now()); err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
}
|
||||
var asker *seat.SharedAskedBy
|
||||
if join || read {
|
||||
if asker, err = askerOf(h.Store, r.Ask); err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) {
|
||||
asker = nil
|
||||
}
|
||||
}
|
||||
event := seat.EventStepStarted
|
||||
if read {
|
||||
event = seat.EventStepEnded
|
||||
}
|
||||
p, _, err := v.change(in.Pipeline, event, func() error {
|
||||
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
|
||||
}, func(p *seat.Pipeline, now time.Time) (int, error) {
|
||||
switch p.Outcome {
|
||||
case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded:
|
||||
return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome)
|
||||
}
|
||||
i, err := stepOf(*p, in.Step, in.Artifact)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
s := &p.Steps[i]
|
||||
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
|
||||
Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now}
|
||||
switch {
|
||||
case ask != nil:
|
||||
s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}}
|
||||
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b)
|
||||
case read:
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
|
||||
s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now
|
||||
if r.Tests != nil {
|
||||
t := *r.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
default:
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
|
||||
if r.Node == "" {
|
||||
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
}
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
}
|
||||
for j := range p.Steps {
|
||||
if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked {
|
||||
p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, ""
|
||||
}
|
||||
}
|
||||
return i, nil
|
||||
})
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
h.pin(p)
|
||||
switch {
|
||||
case join || read:
|
||||
v.Starter.joinRun(run, reader)
|
||||
default:
|
||||
now := h.now()
|
||||
a := *ask
|
||||
a.Run = run
|
||||
// The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and
|
||||
// its readers carried, never put back; a run another step started since is said, not replaced.
|
||||
for {
|
||||
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
|
||||
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
|
||||
for _, o := range r.Readers {
|
||||
if o.Of() == reader {
|
||||
continue
|
||||
}
|
||||
o.Rerun, o.By = !v.passedReader(o, run), ""
|
||||
if o.Rerun {
|
||||
o.By = caller
|
||||
}
|
||||
value.Readers = append(value.Readers, o)
|
||||
}
|
||||
addReaders(&value, []seat.TestRunReader{reader})
|
||||
err := v.Starter.started(a, func() error {
|
||||
if !exists {
|
||||
return h.TestRuns.Create(run, value)
|
||||
}
|
||||
_, err := h.TestRuns.Update(run, value, rev)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) {
|
||||
got, gotRev, gerr := h.TestRuns.Get(run)
|
||||
switch {
|
||||
case errors.Is(gerr, ErrNoRun):
|
||||
r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false
|
||||
continue
|
||||
case gerr != nil:
|
||||
err = gerr
|
||||
case got.Ended == nil:
|
||||
err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask)
|
||||
default:
|
||||
r, rev, exists = got, gotRev, true
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
i, err := stepOf(p, in.Step, in.Artifact)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
res := p.Steps[i]
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
|
||||
}
|
||||
|
||||
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
|
||||
func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
return true // a pipeline gone has no step to move
|
||||
}
|
||||
for _, s := range p.Steps {
|
||||
if s.Artifact == reader.Artifact && sharedRun(s) == run {
|
||||
switch s.Outcome {
|
||||
case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted,
|
||||
seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled:
|
||||
return true
|
||||
}
|
||||
return s.Accepted != nil
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
|
||||
// under one id is kept once), saying it is a rerun and by whom.
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
@@ -325,7 +493,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
} else {
|
||||
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
|
||||
}
|
||||
_, placed := Place(p, reg, half.Entries, now, v.Starter.Name)
|
||||
_, placed := place(p, reg, half.Entries, now, v.Starter.Name, false)
|
||||
base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step)
|
||||
for _, a := range placed {
|
||||
if a.ID != base {
|
||||
|
||||
@@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string)
|
||||
asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{}
|
||||
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Now: h.Now}
|
||||
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts,
|
||||
Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped
|
||||
h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil }
|
||||
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts},
|
||||
store, said, asks, acts, dropped
|
||||
}
|
||||
|
||||
func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult {
|
||||
@@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) {
|
||||
|
||||
// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again
|
||||
// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends
|
||||
// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394).
|
||||
// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as
|
||||
// A-466y amends it).
|
||||
func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) {
|
||||
v, store, said, asks, acts, dropped := verbsFixture()
|
||||
end := func(step, outcome string) {
|
||||
@@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin
|
||||
}
|
||||
// A cancel ends the pipeline cancelled and drops the bounds of the asks still held.
|
||||
c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"})
|
||||
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 {
|
||||
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 {
|
||||
t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped)
|
||||
}
|
||||
if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" {
|
||||
|
||||
@@ -55,9 +55,12 @@ func TestManifest(t *testing.T) {
|
||||
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
|
||||
equalSets(m.Claims[0].Serves, seat.Verbs())},
|
||||
// §27, §85, §330, §355, §77: the old state sections.
|
||||
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})},
|
||||
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})},
|
||||
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
|
||||
{"bounds is key-ttl", keyTTL["bounds"]},
|
||||
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
|
||||
{"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64},
|
||||
{"the test runs' settings", runSettingsOf(raw)},
|
||||
// RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB.
|
||||
{"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256},
|
||||
{"kept 128 MiB", stateMiB[seat.StateKept] == 128},
|
||||
@@ -78,10 +81,13 @@ func TestManifest(t *testing.T) {
|
||||
{"consumes", equalSets(m.Consumes, []string{
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
|
||||
// §647 as A-466z amends it: no holder is read from the seat-holders view.
|
||||
"seat:mesh-module-assigner.views.seat-holders"})},
|
||||
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
|
||||
// asked again by hand (a hand act).
|
||||
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations",
|
||||
@@ -107,7 +113,8 @@ func TestManifest(t *testing.T) {
|
||||
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
|
||||
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
|
||||
// RN25 (§539): the runner's tools carry the prefix runner_.
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
|
||||
"runner_test_runs", "runner_test_runs_drop"})},
|
||||
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
|
||||
// never by a tool of its own; runner_health stays a tool.
|
||||
{"health is the unit", healthTool(raw) == "unit"},
|
||||
@@ -175,3 +182,43 @@ func equalSets(a, b []string) bool {
|
||||
slices.Sort(y)
|
||||
return slices.Equal(x, y)
|
||||
}
|
||||
|
||||
// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a
|
||||
// whole number of at most 64, the three times Go durations, each written into the file its environment names.
|
||||
func runSettingsOf(raw []byte) bool {
|
||||
var m struct {
|
||||
Config map[string]map[string]any `json:"config"`
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Content string `json:"content"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return false
|
||||
}
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"}
|
||||
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) {
|
||||
return false
|
||||
}
|
||||
env := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
for _, v := range r.Env {
|
||||
env[v] = true
|
||||
}
|
||||
}
|
||||
for key, kind := range want {
|
||||
if m.Config[key]["type"] != kind {
|
||||
return false
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path])
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -18,7 +18,9 @@
|
||||
"rerun",
|
||||
"accept",
|
||||
"cancel",
|
||||
"ask-again"
|
||||
"ask-again",
|
||||
"test-runs",
|
||||
"test-runs-drop"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -27,7 +29,9 @@
|
||||
"runner_first_view_again",
|
||||
"runner_health",
|
||||
"runner_unlogged",
|
||||
"runner_unlogged_append"
|
||||
"runner_unlogged_append",
|
||||
"runner_test_runs",
|
||||
"runner_test_runs_drop"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-indexer.take-in.taken",
|
||||
@@ -41,13 +45,17 @@
|
||||
"node-build-go.ask-taken",
|
||||
"node-build-node.ask-taken",
|
||||
"node-build-image.ask-taken",
|
||||
"node-build-go.step-progress",
|
||||
"node-build-node.step-progress",
|
||||
"node-build-image.step-progress",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-conditions.findings-asked",
|
||||
"mesh-delivery.recovery-merged",
|
||||
"mesh-catalog.seat-definition-changed",
|
||||
"mesh-catalog.seat-definition-removed"
|
||||
"mesh-catalog.seat-definition-removed",
|
||||
"seat:mesh-module-assigner.views.seat-holders"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-catalog.seat-definitions",
|
||||
@@ -79,6 +87,34 @@
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"config": {
|
||||
"test-runs-cap-mib": {
|
||||
"type": "number",
|
||||
"description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.",
|
||||
"minimum": 1,
|
||||
"maximum": 64,
|
||||
"trusted": false,
|
||||
"default": 16
|
||||
},
|
||||
"test-runs-kept": {
|
||||
"type": "string",
|
||||
"description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.",
|
||||
"trusted": false,
|
||||
"default": "168h"
|
||||
},
|
||||
"test-step-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "30m"
|
||||
},
|
||||
"run-untaken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
{
|
||||
"name": "pipelines",
|
||||
@@ -107,6 +143,12 @@
|
||||
"history": 1,
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "test-runs",
|
||||
"history": 1,
|
||||
"max-mib": 64,
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "findings",
|
||||
"history": 1
|
||||
@@ -162,6 +204,38 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "test-runs-cap",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-runs-cap-mib",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-runs-cap-mib}"
|
||||
},
|
||||
{
|
||||
"id": "test-runs-kept",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-runs-kept",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-runs-kept}"
|
||||
},
|
||||
{
|
||||
"id": "test-step-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-step-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-step-bound}"
|
||||
},
|
||||
{
|
||||
"id": "run-untaken-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/run-untaken-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "pipeline",
|
||||
"type": "process",
|
||||
@@ -171,7 +245,11 @@
|
||||
"./mesh-delivery-pipeline"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib",
|
||||
"MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept",
|
||||
"MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound",
|
||||
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound"
|
||||
},
|
||||
"user": "mesh-delivery-pipeline",
|
||||
"health": {
|
||||
|
||||
Reference in New Issue
Block a user