Bound every ask the runner puts on a seat (H-A-235)
An ask whose holder lost it (the seat mesh-wants' holder restarted) held its pipeline's turn for nine hours. Every ask but a test run's now arms a key of the state bounds before it is put; its expiry asks once more, said in the journal, and the second ends the step could not run with the finding call-unanswered, so the pipeline ends and its turn frees. Builds: design 47 H H-A-235, H-A-238 (1)-(5)
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// envAskUnansweredBound names the file the manifest writes the setting ask-unanswered-bound into, `${config:<key>}`
|
||||
// (design 47 H H-A-238 (1)).
|
||||
const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE"
|
||||
|
||||
// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default
|
||||
// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration.
|
||||
func askUnansweredBound(getenv func(string) string) (time.Duration, error) {
|
||||
v, err := settingText(getenv(envAskUnansweredBound))
|
||||
if err != nil || v == "" {
|
||||
return pipeline.DefaultAskUnansweredBound, err
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(envAskUnansweredBound), v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to
|
||||
// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered,
|
||||
// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)).
|
||||
func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock {
|
||||
return &pipeline.AskClock{Bound: bound,
|
||||
Arm: func(key string, ttl time.Duration) error {
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil // armed already: the same ask put again at a start keeps its first bound
|
||||
}
|
||||
return err
|
||||
},
|
||||
// A first word is heard for every ask, most of them unbounded or answered already: a key not there is deleted of
|
||||
// nothing and writes no marker.
|
||||
Unbound: func(key string) error {
|
||||
if _, err := bounds.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bounds.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Raise: func(onSeat, ask string, within time.Duration) error {
|
||||
if w == nil {
|
||||
return errors.New("the runner has no finding writer")
|
||||
}
|
||||
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat),
|
||||
Headline: "A step's ask is not answered",
|
||||
Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s, asked it once more, and heard no word of its holder within %s, so its step ended could not run and its pipeline went on (H-A-235).", ask, onSeat, within),
|
||||
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}})
|
||||
},
|
||||
Clear: func(_, ask string) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.Clear(ctx, mcc.ScopeCall, ask, mcc.KindCallUnanswered, "the ask is answered")
|
||||
}}
|
||||
}
|
||||
|
||||
// answered deletes the bounds of the ask on onSeat at its holder's first word, said when it fails: the bound then passes and
|
||||
// its expiry finds the step answered or asks it once more.
|
||||
func answered(c *pipeline.AskClock, onSeat, ask string, logf func(string, ...any)) {
|
||||
if err := c.Answered(onSeat, ask); err != nil {
|
||||
logf("the bounds of ask %s on %s are not deleted at its holder's word: %v", ask, onSeat, err)
|
||||
}
|
||||
}
|
||||
|
||||
// hearWantsProgress takes a step progress of the seat mesh-wants as its holder's first word on the ask it names (H-A-238 (3)).
|
||||
func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
return hub.Handle(wants.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var ev wants.StepProgressEvent
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
logf("%s is no step-progress: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
answered(c, wants.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step.
|
||||
func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool {
|
||||
return func(key string) bool {
|
||||
s, ask, again, ok := pipeline.ParseAskBound(key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if err := unanswered(s, ask, again); err != nil {
|
||||
logf("the bound of ask %s on %s passed and is not taken: %v", ask, s, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -72,8 +72,8 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
|
||||
}
|
||||
}
|
||||
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
|
||||
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
|
||||
// boundExpired takes the expiry of a drop's key of the state bounds: it says the drop unanswered (A-222); an ask's key is
|
||||
// taken by askBoundExpired (H-A-235).
|
||||
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
return func(key string) {
|
||||
if s, ask, ok := pipeline.ParseDropBound(key); ok {
|
||||
|
||||
@@ -67,8 +67,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
|
||||
// The bound of each superseded step's drop and of each ask: a key with its own time to live, its expiry heard (A-222, A-77;
|
||||
// H-A-238 (5)).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop or an ask no holder answers")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -160,6 +161,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
hearer.AskSteps = starter.AskSteps
|
||||
// Every ask but a test run's is bounded until its holder's first word (H-A-235, H-A-238), its bounds keys of the state
|
||||
// bounds.
|
||||
askBound, err := askUnansweredBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("asks: each unanswered ask bound %s, asked once more, then its step ends could not run", askBound)
|
||||
clock := askClock(ctx, bounds, askBound, rv.Findings, cred.Module, time.Now)
|
||||
starter.Asks, hearer.Asks = clock, clock
|
||||
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
@@ -297,10 +307,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return hearer.RunTaken(ask, node)
|
||||
}
|
||||
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
|
||||
if err := hearAskTaken(heard, taken, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := hearWantsProgress(heard, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's write verbs are answered by the process that holds the pipelines (A-107); made before the bounds are heard,
|
||||
// since an ask's expired bound asks it once more through them (H-A-235).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf)
|
||||
expired := func(key string) {
|
||||
if !askExpired(key) {
|
||||
dropExpired(key)
|
||||
}
|
||||
}
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -336,9 +359,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The steps' ends are kept seat events, each on its own stream (step-ended of the build seats, mesh-builds and
|
||||
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
|
||||
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
|
||||
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
stopVerbs, err := answerVerbs(calls, seats, verbs)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+26
-1
@@ -153,6 +153,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askBound := filepath.Join(dir, "ask-unanswered-bound")
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
@@ -164,6 +165,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
return broker
|
||||
case envPipelinesWaitingCap:
|
||||
return waitingCap
|
||||
case envAskUnansweredBound:
|
||||
return askBound
|
||||
}
|
||||
return ""
|
||||
}, logs)
|
||||
@@ -359,9 +362,12 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
}
|
||||
|
||||
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
|
||||
// pipelines-stalled; the start begins no pipeline of its own.
|
||||
// pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second.
|
||||
stopRunner()
|
||||
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
|
||||
if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := pipelineKeys(t, ctx, pipelines)
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeRunning)
|
||||
@@ -370,6 +376,25 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) {
|
||||
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
|
||||
}
|
||||
|
||||
// H-A-235, H-A-238: main.10's build ask, which no holder answers, is asked once more when its bound's key expires, and
|
||||
// at its second bound's expiry its step ends could not run, the pipeline ends failed and frees its turn, and the finding
|
||||
// call-unanswered names the ask.
|
||||
build := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
|
||||
if build.Ask == "" {
|
||||
t.Fatalf("main.10's build = %+v, want it asked", build)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return strings.Contains(logs.String(), "the ask "+build.Ask+" of the build step of main.10 on mesh-builds had no word from its holder within 1s: asked once more")
|
||||
})
|
||||
outcome("main.10", seat.PipelineOutcomeFailed)
|
||||
if s := stepFor(get("main.10"), "app", seat.StepResultStepBuild); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" was not answered" {
|
||||
t.Fatalf("main.10's build = %+v, want could not run, its ask not answered", s)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered")
|
||||
return err == nil && strings.Contains(string(e.Value()), build.Ask) && len(running()) == 0
|
||||
})
|
||||
}
|
||||
|
||||
func stepFor(p seat.Pipeline, module, step string) seat.StepResult {
|
||||
|
||||
@@ -141,6 +141,8 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err)
|
||||
_ = m.Nak()
|
||||
default:
|
||||
// The step's end is its holder's word on the ask, the first for mesh-builds (H-A-238 (3)).
|
||||
answered(h.Asks, bs.Seat, ev.ID, logf)
|
||||
_ = m.Ack()
|
||||
}
|
||||
})
|
||||
@@ -153,8 +155,9 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
|
||||
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
|
||||
// that node's holder, and begins the clock of a test run's ask (§725).
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
for _, ts := range pipeline.TakenSeats {
|
||||
ts := ts
|
||||
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
@@ -167,6 +170,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
// The seat's ask-taken is its holder's first word on the ask (H-A-238 (3)).
|
||||
answered(c, ts.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
@@ -176,8 +181,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
|
||||
// unanswered (A-222, A-77): no timer in the process.
|
||||
// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235,
|
||||
// H-A-238 (5)): no timer in the process.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
|
||||
+5
-3
@@ -350,7 +350,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
|
||||
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
|
||||
}
|
||||
// Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys.
|
||||
// A test run's ask keeps §725's bounds alone (H-A-238 (4)): the state bounds holds no key naming it, only drops' and other
|
||||
// asks' keys.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -359,8 +360,9 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, pipeline.DropBoundPrefix) {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k)
|
||||
_, ask, _, isAsk := pipeline.ParseAskBound(k)
|
||||
if (!isAsk && !strings.HasPrefix(k, pipeline.DropBoundPrefix)) || ask == unit.Ask || ask == integ.Ask {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a test run's ask", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// DefaultAskUnansweredBound is the setting ask-unanswered-bound's default (design 47 H H-A-235, H-A-238 (1)): ten minutes
|
||||
// from an ask until its holder's first word.
|
||||
const DefaultAskUnansweredBound = 10 * time.Minute
|
||||
|
||||
// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask.<seat>.<ask>` armed before the ask is put,
|
||||
// `ask2.<seat>.<ask>` armed when it is asked once more, each with the setting as its own time to live.
|
||||
const (
|
||||
AskBoundPrefix = "ask."
|
||||
AskAgainBoundPrefix = "ask2."
|
||||
)
|
||||
|
||||
// AskBoundKey is the key of the first bound of the ask on onSeat; AskAgainBoundKey that of its second.
|
||||
func AskBoundKey(onSeat, ask string) string { return AskBoundPrefix + onSeat + "." + ask }
|
||||
func AskAgainBoundKey(onSeat, ask string) string { return AskAgainBoundPrefix + onSeat + "." + ask }
|
||||
|
||||
// ParseAskBound reads a key of the state bounds as an ask's bound: the seat, the ask and whether it is the second bound.
|
||||
func ParseAskBound(key string) (onSeat, ask string, again, ok bool) {
|
||||
rest, found := strings.CutPrefix(key, AskAgainBoundPrefix)
|
||||
again = found
|
||||
if !found {
|
||||
if rest, found = strings.CutPrefix(key, AskBoundPrefix); !found {
|
||||
return "", "", false, false
|
||||
}
|
||||
}
|
||||
onSeat, ask, found = strings.Cut(rest, ".")
|
||||
return onSeat, ask, again, found && onSeat != "" && ask != "" && !strings.Contains(ask, ".")
|
||||
}
|
||||
|
||||
// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235).
|
||||
func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" }
|
||||
|
||||
// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)):
|
||||
// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding
|
||||
// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)).
|
||||
type AskClock struct {
|
||||
Bound time.Duration
|
||||
Arm func(key string, ttl time.Duration) error
|
||||
Unbound func(key string) error
|
||||
Raise func(onSeat, ask string, bound time.Duration) error
|
||||
Clear func(onSeat, ask string) error
|
||||
}
|
||||
|
||||
// arm arms the first bound of a, before it is put (H-A-238 (5)); a test run's ask, and a nil clock, arm none.
|
||||
func (c *AskClock) arm(a Asked) error {
|
||||
if c == nil || c.Arm == nil || a.Run != "" {
|
||||
return nil
|
||||
}
|
||||
if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.bound()); err != nil {
|
||||
return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *AskClock) bound() time.Duration {
|
||||
if c.Bound > 0 {
|
||||
return c.Bound
|
||||
}
|
||||
return DefaultAskUnansweredBound
|
||||
}
|
||||
|
||||
// Answered deletes both bounds of the ask on onSeat, its holder's first word heard: an ask-taken, a step progress or the
|
||||
// step's end (H-A-238 (3)).
|
||||
func (c *AskClock) Answered(onSeat, ask string) error {
|
||||
if c == nil || c.Unbound == nil || ask == "" {
|
||||
return nil
|
||||
}
|
||||
return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)))
|
||||
}
|
||||
|
||||
// answeredLate clears the finding of the ask of step s, which ended unanswered, its holder heard after all (H-A-238 (2)).
|
||||
func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) {
|
||||
if c == nil || c.Clear == nil || s.Ask == "" || s.Why != UnansweredWhy(s.Ask) {
|
||||
return
|
||||
}
|
||||
if err := c.Clear(s.Seat, s.Ask); err != nil {
|
||||
logf("the finding of ask %s, answered after its step ended, is not cleared: %v", s.Ask, err)
|
||||
}
|
||||
}
|
||||
|
||||
// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235): at the first the ask is put once more under its
|
||||
// own id, said in the journal, its second bound armed; at the second the step ends could not run "its ask <id> was not
|
||||
// answered", the finding call-unanswered raised, so the pipeline ends and its turn frees. A bound whose ask no running step
|
||||
// names any longer asks nothing.
|
||||
func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
|
||||
h, c := v.Hearer, v.Starter.Asks
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
key, p, i, err := v.runningAsk(onSeat, ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
s := p.Steps[i]
|
||||
if again {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within its two bounds: the step ends could not run (H-A-235)",
|
||||
ask, s.Step, key, onSeat)
|
||||
if err := h.endUnanswered(key, ask); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Raise != nil {
|
||||
return c.Raise(onSeat, ask, 2*c.bound())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
a, err := v.askAgain(p, s, h.now())
|
||||
if err != nil {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", ask, s.Step, key, onSeat, c.bound(), err)
|
||||
} else {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more (H-A-235)", ask, s.Step, key, onSeat, c.bound())
|
||||
}
|
||||
if c.Arm != nil {
|
||||
if aerr := c.Arm(AskAgainBoundKey(onSeat, ask), c.bound()); aerr != nil {
|
||||
return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", ask, aerr)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil // its second bound ends the step
|
||||
}
|
||||
v.Starter.carry(key, []Asked{*a})
|
||||
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
|
||||
}
|
||||
|
||||
// runningAsk is the pipeline and the index of its running step asked under ask of onSeat; an empty key when none is.
|
||||
func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) {
|
||||
pkeys, err := allKeys(v.Hearer.Store)
|
||||
if err != nil {
|
||||
return "", seat.Pipeline{}, -1, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 && open(p.Steps[i]) && p.Steps[i].Seat == onSeat {
|
||||
return key, p, i, nil
|
||||
}
|
||||
}
|
||||
return "", seat.Pipeline{}, -1, nil
|
||||
}
|
||||
|
||||
// askAgain is the ask of running step s as it was put: its own id, a rerun's marks kept.
|
||||
func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) {
|
||||
a, err := v.placedAsk(p, s, s.Ask, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.Rerun {
|
||||
markRerun(a, s.By)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// markRerun marks a's body a rerun by caller.
|
||||
func markRerun(a *Asked, caller string) {
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
}
|
||||
}
|
||||
|
||||
// endUnanswered ends the running step of pipeline key asked under ask could not run, its ask not answered (H-A-235): every
|
||||
// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when
|
||||
// nothing else is open, said and logged as any step's end.
|
||||
func (h *Hearer) endUnanswered(key, ask string) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || !open(p.Steps[i]) {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, UnansweredWhy(ask), &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
|
||||
var readers []int
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = sameValidate(&p, i, now)
|
||||
_, notAsked := releaseCompositions(&p, now)
|
||||
readers = append(readers, notAsked...)
|
||||
}
|
||||
asked, err := askApproval(key, &p, now, h.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended := h.endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
for _, j := range append([]int{i}, readers...) {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks
|
||||
// once more under the ask's own id, its second bound armed; past the second the step ends could not run "its ask <id> was not
|
||||
// answered", the finding raised, and the pipeline ends; a first word deletes both bounds, and a late one clears the finding.
|
||||
func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
|
||||
v, store, said, asks, _, _ := verbsFixture()
|
||||
var armed, unbound, raised, cleared []string
|
||||
logs := []string{}
|
||||
v.Hearer.Logf = func(f string, a ...any) { logs = append(logs, f) }
|
||||
c := &AskClock{Bound: 10 * time.Minute,
|
||||
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
|
||||
Unbound: func(key string) error { unbound = append(unbound, key); return nil },
|
||||
Raise: func(s, ask string, _ time.Duration) error { raised = append(raised, s+" "+ask); return nil },
|
||||
Clear: func(s, ask string) error { cleared = append(cleared, s+" "+ask); return nil }}
|
||||
v.Starter.Asks, v.Hearer.Asks = c, c
|
||||
|
||||
// A test run's ask keeps §725's bounds; a nil clock arms none.
|
||||
if err := c.arm(Asked{Seat: nodebuildgo.Seat, ID: "r1", Run: "run"}); err != nil || len(armed) != 0 {
|
||||
t.Fatalf("a test run's ask armed %v (%v)", armed, err)
|
||||
}
|
||||
if err := (*AskClock)(nil).arm(Asked{Seat: nodebuildgo.Seat, ID: "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Every step but app:program's lint has ended; that lint failed, and its rerun's ask is armed before it is put.
|
||||
p, rev, _ := store.Get(pkey)
|
||||
for i := range p.Steps {
|
||||
if open(p.Steps[i]) {
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomePassed
|
||||
}
|
||||
}
|
||||
p.Steps[stepIndex(p, "lint", "app:program")].Outcome = seat.StepResultOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := r.Result.Ask
|
||||
if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 {
|
||||
t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want)
|
||||
}
|
||||
// The first bound passes: asked once more under its own id, a rerun still, the second bound armed and said.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asks.ids) != 2 || asks.ids[1] != ask || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, ask)+" 10m0s" {
|
||||
t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, ask)
|
||||
}
|
||||
if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" {
|
||||
t.Errorf("the ask put again = %+v", asks.asks[1])
|
||||
}
|
||||
if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) {
|
||||
t.Errorf("no journal line names the ask asked again: %v", logs)
|
||||
}
|
||||
if got, _, _ := store.Get(pkey); got.Steps[stepIndex(got, "lint", "app:program")].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatal("the step ended at the first bound")
|
||||
}
|
||||
// The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
s := got.Steps[stepIndex(got, "lint", "app:program")]
|
||||
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+ask+" was not answered" || s.Ended == nil {
|
||||
t.Errorf("the step = %+v", s)
|
||||
}
|
||||
if got.Outcome != seat.PipelineOutcomeFailed || len(asks.ids) != 2 {
|
||||
t.Errorf("the pipeline is %s (%s), asked %d, want failed and nothing more asked", got.Outcome, got.Why, len(asks.ids))
|
||||
}
|
||||
if !slices.Equal(raised, []string{nodebuildgo.Seat + " " + ask}) || !slices.Contains(*said, "pipeline-ended.main.5") {
|
||||
t.Errorf("raised %v, said %v", raised, *said)
|
||||
}
|
||||
// A bound whose step ended asks nothing; a first word deletes both bounds; a late end clears the finding.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 {
|
||||
t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err)
|
||||
}
|
||||
if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask)}) {
|
||||
t.Errorf("a first word deleted %v (%v)", unbound, err)
|
||||
}
|
||||
if err := v.Hearer.Taken(ask, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) {
|
||||
t.Errorf("a late ask-taken cleared %v (%v)", cleared, err)
|
||||
}
|
||||
for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} {
|
||||
if s, a, again, ok := ParseAskBound(k); !ok || s != "mesh-wants" || a != "a1" || again != strings.HasPrefix(k, AskAgainBoundPrefix) {
|
||||
t.Errorf("ParseAskBound(%s) = %s %s %v %v", k, s, a, again, ok)
|
||||
}
|
||||
}
|
||||
if _, _, _, ok := ParseAskBound(DropBoundKey("node-build-go", "a1")); ok {
|
||||
t.Error("a drop's bound read as an ask's")
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
|
||||
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder is bounded by ask-unanswered-bound
|
||||
// (H-A-235, H-A-238; AskClock), a test run's by run-untaken-bound (§725).
|
||||
const DropBound = 5 * time.Minute
|
||||
|
||||
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
|
||||
|
||||
@@ -387,17 +387,23 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
|
||||
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error {
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms
|
||||
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put.
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) {
|
||||
accept = to
|
||||
}
|
||||
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
if ok {
|
||||
var err error
|
||||
switch {
|
||||
case !ok:
|
||||
err = fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
case arm != nil:
|
||||
err = arm(a)
|
||||
}
|
||||
if err == nil {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
|
||||
@@ -183,6 +183,8 @@ type Starter struct {
|
||||
// Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil
|
||||
// places every start at once.
|
||||
Queue *Queue
|
||||
// Asks bounds every ask but a test run's until its holder's first word (H-A-235, H-A-238); nil bounds none.
|
||||
Asks *AskClock
|
||||
// RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places
|
||||
// it at the runner's next start.
|
||||
RunTurn func(UnplacedRun)
|
||||
@@ -252,7 +254,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error {
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) })
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm)
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
|
||||
@@ -176,6 +176,9 @@ type Hearer struct {
|
||||
// Unplaced says why a mover of a pipeline has no step placed, for the why of its end (H-A-171; Starter.Unplaced): it
|
||||
// waits on its registration or on a publish; nil says only that it has no step placed.
|
||||
Unplaced func(p seat.Pipeline, module string) string
|
||||
// Asks clears the finding of an ask that ended its step unanswered once it is answered after all (H-A-238 (2)); nil
|
||||
// clears none.
|
||||
Asks *AskClock
|
||||
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
|
||||
// nil keys none.
|
||||
Rejoin func(key string)
|
||||
@@ -249,6 +252,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
|
||||
case !open(p.Steps[i]):
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
|
||||
@@ -363,7 +363,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
break
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i >= 0 && !open(p.Steps[i]) {
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
}
|
||||
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
@@ -321,7 +320,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
}
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -468,7 +467,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -500,8 +499,24 @@ func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
}
|
||||
|
||||
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
|
||||
// under one id is kept once), saying it is a rerun and by whom.
|
||||
// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover
|
||||
// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236).
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step)
|
||||
if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate {
|
||||
id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate)
|
||||
}
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
markRerun(a, caller)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its
|
||||
// mover (H-A-236), any other the one Place makes for it.
|
||||
func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
|
||||
if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") {
|
||||
return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask")
|
||||
}
|
||||
@@ -509,19 +524,13 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module)
|
||||
}
|
||||
// The validate ask is the pipeline's one, naming no mover (H-A-51): its rerun asks that one ask again; a composition's
|
||||
// rerun builds the mover's composition ask directly, never looked up among the placed steps (H-A-236).
|
||||
switch s.Step {
|
||||
case seat.StepResultStepValidate, seat.StepResultStepComposition:
|
||||
if s.Seat == wants.Seat {
|
||||
id, step := AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate), wants.StepRunStepValidate
|
||||
if s.Step == seat.StepResultStepComposition {
|
||||
id, step = AskID(now, p.Pipeline, m.Module, "", seat.StepResultStepComposition), wants.StepRunStepComposition
|
||||
}
|
||||
a := wantsAsk(p, m, id, step)
|
||||
b := a.Ask.(wants.StepRun)
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
if s.Seat == wants.Seat {
|
||||
switch s.Step {
|
||||
case seat.StepResultStepValidate:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepValidate)
|
||||
return &a, nil
|
||||
case seat.StepResultStepComposition:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepComposition)
|
||||
return &a, nil
|
||||
}
|
||||
}
|
||||
@@ -549,21 +558,9 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if a.ID != base {
|
||||
continue
|
||||
}
|
||||
a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step)
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
b.ID = a.ID
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.ID = id
|
||||
if b, ok := a.Ask.(nodebuildgo.StepAsk); ok {
|
||||
b.ID = id
|
||||
a.Ask = b
|
||||
}
|
||||
return &a, nil
|
||||
|
||||
@@ -84,7 +84,7 @@ func TestManifest(t *testing.T) {
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
|
||||
@@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool {
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return false
|
||||
}
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
|
||||
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
|
||||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
"node-build-node.step-progress",
|
||||
"node-build-image.step-progress",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.version-released",
|
||||
@@ -151,6 +152,12 @@
|
||||
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
},
|
||||
"ask-unanswered-bound": {
|
||||
"type": "string",
|
||||
"description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).",
|
||||
"trusted": false,
|
||||
"default": "10m"
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
@@ -278,6 +285,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "ask-unanswered-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-unanswered-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-unanswered-bound}"
|
||||
},
|
||||
{
|
||||
"id": "widens-read-bound",
|
||||
"type": "file",
|
||||
@@ -326,6 +341,7 @@
|
||||
"MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound",
|
||||
"MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept",
|
||||
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
|
||||
"MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound",
|
||||
"MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once",
|
||||
"MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user