Bound a silent seat's ask once; put it again when its holder starts (H-A-243)

mesh-builds and mesh-wants say nothing before a step's end, so asking
them again at the short bound would double a running build. Their ask
has one bound, ask-silent-bound, and is put again when the node-engine
says the holder running, since the holder takes asks on a plain
subscription and loses those put while it is down. An ask asked again
on a build seat carries a second id derived in the accepted form.

Builds: design 47 H H-A-235, H-A-238, H-A-241, H-A-243
This commit is contained in:
2026-11-21 12:02:02 +01:00
parent b95b3ef989
commit 902947a044
10 changed files with 347 additions and 56 deletions
@@ -9,6 +9,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
@@ -20,16 +21,28 @@ import (
// (design 47 H H-A-238 (1)).
const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE"
// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)).
const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE"
// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default
// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration.
func askUnansweredBound(getenv func(string) string) (time.Duration, error) {
v, err := settingText(getenv(envAskUnansweredBound))
return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound)
}
// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)).
func askSilentBound(getenv func(string) string) (time.Duration, error) {
return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound)
}
func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) {
v, err := settingText(getenv(env))
if err != nil || v == "" {
return pipeline.DefaultAskUnansweredBound, err
return def, err
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(envAskUnansweredBound), v)
return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v)
}
return d, nil
}
@@ -37,8 +50,8 @@ func askUnansweredBound(getenv func(string) string) (time.Duration, error) {
// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to
// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered,
// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)).
func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock {
return &pipeline.AskClock{Bound: bound,
func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock {
return &pipeline.AskClock{Bound: bound, Silent: silent,
Arm: func(key string, ttl time.Duration) error {
_, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl))
if errors.Is(err, jetstream.ErrKeyExists) {
@@ -66,7 +79,7 @@ func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duratio
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat),
Headline: "A step's ask is not answered",
Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s, asked it once more, and heard no word of its holder within %s, so its step ended could not run and its pipeline went on (H-A-235).", ask, onSeat, within),
Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within),
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}})
},
Clear: func(_, ask string) error {
@@ -99,6 +112,31 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...
})
}
// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)).
func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error {
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
var ev nodeengine.Running
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" {
logf("%s is no running: %v", m.Subject(), err)
_ = m.Term()
return
}
if ev.Stopped {
_ = m.Ack()
return
}
n, err := started(ev.Module)
switch {
case errors.Is(err, pipeline.ErrHoldersNotRead):
_ = m.Nak() // heard again once the view is read
return
case err != nil:
logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err)
}
_ = m.Ack()
})
}
// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step.
func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool {
return func(key string) bool {
@@ -167,8 +167,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
logger.Printf("asks: each unanswered ask bound %s, asked once more, then its step ends could not run", askBound)
clock := askClock(ctx, bounds, askBound, rv.Findings, cred.Module, time.Now)
silentBound, err := askSilentBound(getenv)
if err != nil {
return err
}
logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound)
clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now)
starter.Asks, hearer.Asks = clock, clock
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
@@ -317,6 +321,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// since an ask's expired bound asks it once more through them (H-A-235).
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil {
return err
}
dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf)
expired := func(key string) {
@@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
const router = "operator-router"
answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{
map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}})
// Each mesh-builds step ask as its holder hears it, on its plain subscription.
var buildAsksMu sync.Mutex
buildAsks := map[string]int{}
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
buildAsksMu.Lock()
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
buildAsksMu.Unlock()
}); err != nil {
t.Fatal(err)
}
buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] }
records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1})
if err != nil {
t.Fatal(err)
@@ -154,6 +165,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
t.Fatal(err)
}
askBound := filepath.Join(dir, "ask-unanswered-bound")
silentBound := filepath.Join(dir, "ask-silent-bound")
logs := &syncBuffer{}
start := func() func() {
runCtx, stop := context.WithCancel(ctx)
@@ -167,6 +179,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
return waitingCap
case envAskUnansweredBound:
return askBound
case envAskSilentBound:
return silentBound
}
return ""
}, logs)
@@ -362,12 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
}
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
// pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second.
// pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its
// ask-silent-bound four.
stopRunner()
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil {
t.Fatal(err)
}
before := pipelineKeys(t, ctx, pipelines)
stopRunner = start()
outcome("main.10", seat.PipelineOutcomeRunning)
@@ -377,19 +395,21 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
}
// H-A-235, H-A-238: main.10's build ask, which no holder answers, is asked once more when its bound's key expires, and
// at its second bound's expiry its step ends could not run, the pipeline ends failed and frees its turn, and the finding
// call-unanswered names the ask.
// H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for
// ask-unanswered-bound (a build running four of those bounds is not touched); its one bound,
// ask-silent-bound, ends its step could not run "its ask <id> had no end", the pipeline ending failed, its turn freed and
// the finding call-unanswered naming the ask.
build := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
if build.Ask == "" {
if build.Ask == "" || build.Started == nil {
t.Fatalf("main.10's build = %+v, want it asked", build)
}
waitFor(t, ctx, logs, func() bool {
return strings.Contains(logs.String(), "the ask "+build.Ask+" of the build step of main.10 on mesh-builds had no word from its holder within 1s: asked once more")
})
outcome("main.10", seat.PipelineOutcomeFailed)
if s := stepFor(get("main.10"), "app", seat.StepResultStepBuild); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" was not answered" {
t.Fatalf("main.10's build = %+v, want could not run, its ask not answered", s)
s := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second {
t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s)
}
if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 {
t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask)))
}
waitFor(t, ctx, logs, func() bool {
e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered")
@@ -16,6 +16,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
t.Fatal(err)
}
// mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end.
if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil {
t.Fatal(err)
}
var buildAsksMu sync.Mutex
buildAsks := map[string]int{}
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
buildAsksMu.Lock()
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
buildAsksMu.Unlock()
}); err != nil {
t.Fatal(err)
}
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
// integration step optional; web is built by node-build-node.
@@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
const p3 = "main.1"
takeIn("app", "c1", "src2:aa", 3, 1)
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
// H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while
// it was down is lost; a running of a module holding no silent seat puts nothing again.
firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask
publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()})
publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()})
waitFor(t, ctx, logs, func() bool {
buildAsksMu.Lock()
n := buildAsks[firstBuild]
buildAsksMu.Unlock()
return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1
})
passBuild(p3, "app", "c1", "src2:aa")
unitRun, unit := runOfStep(p3, "app:program", "unit")
integRun, integ := runOfStep(p3, "app:program", "integration")
+1 -1
View File
@@ -10,7 +10,7 @@ require (
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
@@ -1,6 +1,8 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"strings"
@@ -16,6 +18,24 @@ import (
// from an ask until its holder's first word.
const DefaultAskUnansweredBound = 10 * time.Minute
// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose
// holder says no word before the step's end (mesh-builds, mesh-wants) until that end.
const DefaultAskSilentBound = 45 * time.Minute
// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the
// build seats, which say ask-taken (H-A-241, H-A-243).
func Silent(onSeat string) bool { return !BoundSeat(onSeat) }
// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the
// first id and ".2", the form the holders accept, so no duplicate window is relied on.
func AgainID(id string) string {
if len(id) <= 6 {
return id
}
sum := sha256.Sum256([]byte(id + ".2"))
return id[:len(id)-6] + hex.EncodeToString(sum[:3])
}
// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask.<seat>.<ask>` armed before the ask is put,
// `ask2.<seat>.<ask>` armed when it is asked once more, each with the setting as its own time to live.
const (
@@ -43,11 +63,16 @@ func ParseAskBound(key string) (onSeat, ask string, again, ok bool) {
// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235).
func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" }
// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)).
func SilentWhy(ask string) string { return "its ask " + ask + " had no end" }
// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)):
// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding
// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)).
type AskClock struct {
// Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat.
Bound time.Duration
Silent time.Duration
Arm func(key string, ttl time.Duration) error
Unbound func(key string) error
Raise func(onSeat, ask string, bound time.Duration) error
@@ -59,12 +84,23 @@ func (c *AskClock) arm(a Asked) error {
if c == nil || c.Arm == nil || a.Run != "" {
return nil
}
if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.bound()); err != nil {
if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil {
return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err)
}
return nil
}
// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound.
func (c *AskClock) boundOf(onSeat string) time.Duration {
if !Silent(onSeat) {
return c.bound()
}
if c.Silent > 0 {
return c.Silent
}
return DefaultAskSilentBound
}
func (c *AskClock) bound() time.Duration {
if c.Bound > 0 {
return c.Bound
@@ -72,18 +108,21 @@ func (c *AskClock) bound() time.Duration {
return DefaultAskUnansweredBound
}
// Answered deletes both bounds of the ask on onSeat, its holder's first word heard: an ask-taken, a step progress or the
// step's end (H-A-238 (3)).
// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a
// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under
// AgainID goes too.
func (c *AskClock) Answered(onSeat, ask string) error {
if c == nil || c.Unbound == nil || ask == "" {
return nil
}
return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)))
return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)),
c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask))))
}
// answeredLate clears the finding of the ask of step s, which ended unanswered, its holder heard after all (H-A-238 (2)).
// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all
// (H-A-238 (2)).
func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) {
if c == nil || c.Clear == nil || s.Ask == "" || s.Why != UnansweredWhy(s.Ask) {
if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) {
return
}
if err := c.Clear(s.Seat, s.Ask); err != nil {
@@ -91,10 +130,12 @@ func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) {
}
}
// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235): at the first the ask is put once more under its
// own id, said in the journal, its second bound armed; at the second the step ends could not run "its ask <id> was not
// answered", the finding call-unanswered raised, so the pipeline ends and its turn frees. A bound whose ask no running step
// names any longer asks nothing.
// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound,
// ask-silent-bound, ends the step could not run "its ask <id> had no end", never asking again (that would double a running
// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its
// second bound armed under that id; the second ends the step could not run "its ask <id> was not answered". An ended step
// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any
// longer asks nothing.
func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
h, c := v.Hearer, v.Starter.Asks
if c == nil {
@@ -105,26 +146,30 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
return err
}
s := p.Steps[i]
if again {
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within its two bounds: the step ends could not run (H-A-235)",
ask, s.Step, key, onSeat)
if err := h.endUnanswered(key, ask); err != nil {
if Silent(onSeat) || again {
why, within := UnansweredWhy(s.Ask), 2*c.bound()
if Silent(onSeat) {
why, within = SilentWhy(s.Ask), c.boundOf(onSeat)
}
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)",
s.Ask, s.Step, key, onSeat, within)
if err := h.endUnanswered(key, s.Ask, why); err != nil {
return err
}
if c.Raise != nil {
return c.Raise(onSeat, ask, 2*c.bound())
return c.Raise(onSeat, s.Ask, within)
}
return nil
}
a, err := v.askAgain(p, s, h.now())
a, err := v.askAgain(p, s, AgainID(s.Ask), h.now())
if err != nil {
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", ask, s.Step, key, onSeat, c.bound(), err)
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err)
} else {
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more (H-A-235)", ask, s.Step, key, onSeat, c.bound())
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID)
}
if c.Arm != nil {
if aerr := c.Arm(AskAgainBoundKey(onSeat, ask), c.bound()); aerr != nil {
return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", ask, aerr)
if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil {
return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr)
}
}
if err != nil {
@@ -134,7 +179,83 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
}
// runningAsk is the pipeline and the index of its running step asked under ask of onSeat; an empty key when none is.
// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain
// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under
// its own id, a rerun's marks kept, said in the journal; its bound stands as armed.
func (v Verbs) HolderStarted(module string) (int, error) {
st := v.Starter
if st == nil || st.Holders == nil {
return 0, nil
}
var silent []string
for onSeat := range acceptStep {
if !Silent(onSeat) {
continue
}
holders, read := st.Holders(onSeat)
if !read {
return 0, ErrHoldersNotRead
}
if containsString(holders, module) {
silent = append(silent, onSeat)
}
}
if len(silent) == 0 {
return 0, nil
}
pkeys, err := allKeys(v.Hearer.Store)
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now, asked := v.Hearer.now(), 0
var first error
for _, key := range pkeys {
p, _, err := v.Hearer.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning {
continue
}
seen := map[string]bool{}
var due []Asked
for _, s := range p.Steps {
if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) {
continue
}
seen[s.Ask] = true
a, err := v.askAgain(p, s, s.Ask, now)
if err != nil {
v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err)
continue
}
v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)",
s.Ask, s.Step, key, s.Seat, module)
due = append(due, *a)
}
if len(due) == 0 {
continue
}
st.carry(key, due)
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil {
first = err
}
asked += len(due)
}
return asked, first
}
// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again.
var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet")
func containsString(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty
// key when none is.
func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) {
pkeys, err := allKeys(v.Hearer.Store)
if err != nil {
@@ -152,9 +273,9 @@ func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error
return "", seat.Pipeline{}, -1, nil
}
// askAgain is the ask of running step s as it was put: its own id, a rerun's marks kept.
func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) {
a, err := v.placedAsk(p, s, s.Ask, now)
// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept.
func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
a, err := v.placedAsk(p, s, id, now)
if err != nil {
return nil, err
}
@@ -183,10 +304,10 @@ func markRerun(a *Asked, caller string) {
}
}
// endUnanswered ends the running step of pipeline key asked under ask could not run, its ask not answered (H-A-235): every
// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every
// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when
// nothing else is open, said and logged as any step's end.
func (h *Hearer) endUnanswered(key, ask string) error {
func (h *Hearer) endUnanswered(key, ask, why string) error {
for {
p, rev, err := h.Store.Get(key)
if err != nil {
@@ -198,7 +319,7 @@ func (h *Hearer) endUnanswered(key, ask string) error {
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, UnansweredWhy(ask), &now
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
var readers []int
if s.Step == seat.StepResultStepValidate {
@@ -6,8 +6,10 @@ import (
"testing"
"time"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks
@@ -50,14 +52,19 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 {
t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want)
}
// The first bound passes: asked once more under its own id, a rerun still, the second bound armed and said.
// The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun
// still, the second bound armed under it and said (H-A-243 (3)).
again := AgainID(ask)
if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] {
t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err)
}
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil {
t.Fatal(err)
}
if len(asks.ids) != 2 || asks.ids[1] != ask || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, ask)+" 10m0s" {
t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, ask)
if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" {
t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again)
}
if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" {
if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" {
t.Errorf("the ask put again = %+v", asks.asks[1])
}
if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) {
@@ -67,7 +74,7 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
t.Fatal("the step ended at the first bound")
}
// The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed.
if err := v.AskUnanswered(nodebuildgo.Seat, ask, true); err != nil {
if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil {
t.Fatal(err)
}
got, _, _ := store.Get(pkey)
@@ -85,10 +92,10 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 {
t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err)
}
if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask)}) {
if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) {
t.Errorf("a first word deleted %v (%v)", unbound, err)
}
if err := v.Hearer.Taken(ask, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) {
if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) {
t.Errorf("a late ask-taken cleared %v (%v)", cleared, err)
}
for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} {
@@ -100,3 +107,59 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
t.Error("a drop's bound read as an ask's")
}
}
// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound:
// its one bound, ask-silent-bound, ends the step could not run "its ask <id> had no end", the finding raised. The node-engine's
// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none.
func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) {
v, store, _, asks, _, _ := verbsFixture()
var armed, raised []string
c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute,
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
Raise: func(s, ask string, within time.Duration) error {
raised = append(raised, s+" "+ask+" "+within.String())
return nil
}}
v.Starter.Asks, v.Hearer.Asks = c, c
v.Starter.Holders = func(s string) ([]string, bool) {
return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true
}
p, rev, _ := store.Get(pkey)
// Every step has ended but app's build, asked of mesh-builds.
b := stepIndex(p, seat.StepResultStepBuild, "")
for i := range p.Steps {
if open(p.Steps[i]) && i != b {
p.Steps[i].Outcome = seat.StepResultOutcomePassed
}
}
if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) {
t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b])
}
store.Update(pkey, p, rev)
build := p.Steps[b].Ask
if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) {
t.Fatalf("armed %v (%v), want the silent bound", armed, err)
}
// A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id.
if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 {
t.Fatalf("another holder's start put %d (%v)", n, err)
}
if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build {
t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build)
}
if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep {
t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0])
}
// The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends.
if err := v.AskUnanswered(builds.Seat, build, false); err != nil {
t.Fatal(err)
}
got, _, _ := store.Get(pkey)
s := got.Steps[b]
if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed {
t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome)
}
if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) {
t.Errorf("raised %v", raised)
}
}
@@ -589,7 +589,8 @@ func endOutcome(o string) bool {
func stepOfAsk(p seat.Pipeline, id string) int {
for i, s := range p.Steps {
if id != "" && s.Ask == id {
// An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's.
if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) {
return i
}
}
@@ -84,7 +84,7 @@ func TestManifest(t *testing.T) {
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "mesh-catalog.registration-changed",
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
@@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool {
if json.Unmarshal(raw, &m) != nil {
return false
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
@@ -59,6 +59,7 @@
"node-build-image.step-progress",
"mesh-wants.step-ended",
"mesh-wants.step-progress",
"node-engine.running",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision",
"mesh-catalog.version-released",
@@ -158,6 +159,12 @@
"description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).",
"trusted": false,
"default": "10m"
},
"ask-silent-bound": {
"type": "string",
"description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).",
"trusted": false,
"default": "45m"
}
},
"state": [
@@ -293,6 +300,14 @@
"mode": "0600",
"content": "${config:ask-unanswered-bound}"
},
{
"id": "ask-silent-bound",
"type": "file",
"path": "${dir:state}/ask-silent-bound",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:ask-silent-bound}"
},
{
"id": "widens-read-bound",
"type": "file",
@@ -342,6 +357,7 @@
"MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept",
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
"MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound",
"MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound",
"MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once",
"MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap"
},