From 902947a044d8d7d23277adeca4eb50054360bfb6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:42:43 +0100 Subject: [PATCH] Bound a silent seat's ask once; put it again when its holder starts (H-A-243) mesh-builds and mesh-wants say nothing before a step's end, so asking them again at the short bound would double a running build. Their ask has one bound, ask-silent-bound, and is put again when the node-engine says the holder running, since the holder takes asks on a plain subscription and loses those put while it is down. An ask asked again on a build seat carries a second id derived in the accepted form. Builds: design 47 H H-A-235, H-A-238, H-A-241, H-A-243 --- .../cmd/mesh-delivery-pipeline/askbounds.go | 50 ++++- .../cmd/mesh-delivery-pipeline/main.go | 11 +- .../queue_integration_test.go | 40 +++- .../testruns_integration_test.go | 25 +++ modules/mesh-delivery-pipeline/go.mod | 2 +- .../internal/pipeline/asks.go | 175 +++++++++++++++--- .../internal/pipeline/asks_test.go | 77 +++++++- .../internal/pipeline/steps.go | 3 +- .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 16 ++ 10 files changed, 347 insertions(+), 56 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go index b91e9069..94d7323e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -9,6 +9,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" @@ -20,16 +21,28 @@ import ( // (design 47 H H-A-238 (1)). const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE" +// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)). +const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE" + // askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default // ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration. func askUnansweredBound(getenv func(string) string) (time.Duration, error) { - v, err := settingText(getenv(envAskUnansweredBound)) + return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound) +} + +// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)). +func askSilentBound(getenv func(string) string) (time.Duration, error) { + return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound) +} + +func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) { + v, err := settingText(getenv(env)) if err != nil || v == "" { - return pipeline.DefaultAskUnansweredBound, err + return def, err } d, err := time.ParseDuration(v) if err != nil || d <= 0 { - return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(envAskUnansweredBound), v) + return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v) } return d, nil } @@ -37,8 +50,8 @@ func askUnansweredBound(getenv func(string) string) (time.Duration, error) { // askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to // live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered, // scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)). -func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { - return &pipeline.AskClock{Bound: bound, +func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { + return &pipeline.AskClock{Bound: bound, Silent: silent, Arm: func(key string, ttl time.Duration) error { _, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl)) if errors.Is(err, jetstream.ErrKeyExists) { @@ -66,7 +79,7 @@ func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duratio return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat), Headline: "A step's ask is not answered", - Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s, asked it once more, and heard no word of its holder within %s, so its step ended could not run and its pipeline went on (H-A-235).", ask, onSeat, within), + Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within), Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}}) }, Clear: func(_, ask string) error { @@ -99,6 +112,31 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ... }) } +// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). +func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { + return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { + var ev nodeengine.Running + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" { + logf("%s is no running: %v", m.Subject(), err) + _ = m.Term() + return + } + if ev.Stopped { + _ = m.Ack() + return + } + n, err := started(ev.Module) + switch { + case errors.Is(err, pipeline.ErrHoldersNotRead): + _ = m.Nak() // heard again once the view is read + return + case err != nil: + logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) + } + _ = m.Ack() + }) +} + // askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step. func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool { return func(key string) bool { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 378dc72d..f5f8af67 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -167,8 +167,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - logger.Printf("asks: each unanswered ask bound %s, asked once more, then its step ends could not run", askBound) - clock := askClock(ctx, bounds, askBound, rv.Findings, cred.Module, time.Now) + silentBound, err := askSilentBound(getenv) + if err != nil { + return err + } + logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound) + clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now) starter.Asks, hearer.Asks = clock, clock starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } @@ -317,6 +321,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // since an ask's expired bound asks it once more through them (H-A-235). verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil { + return err + } dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf) expired := func(key string) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 5ad1481c..84a5e92a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { const router = "operator-router" answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{ map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}}) + // Each mesh-builds step ask as its holder hears it, on its plain subscription. + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } + buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] } records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1}) if err != nil { t.Fatal(err) @@ -154,6 +165,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { t.Fatal(err) } askBound := filepath.Join(dir, "ask-unanswered-bound") + silentBound := filepath.Join(dir, "ask-silent-bound") logs := &syncBuffer{} start := func() func() { runCtx, stop := context.WithCancel(ctx) @@ -167,6 +179,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { return waitingCap case envAskUnansweredBound: return askBound + case envAskSilentBound: + return silentBound } return "" }, logs) @@ -362,12 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } // At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears - // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second. + // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its + // ask-silent-bound four. stopRunner() setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1) if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil { t.Fatal(err) } + if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil { + t.Fatal(err) + } before := pipelineKeys(t, ctx, pipelines) stopRunner = start() outcome("main.10", seat.PipelineOutcomeRunning) @@ -377,19 +395,21 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { t.Fatalf("the start wrote pipelines %v, want %v", after, before) } - // H-A-235, H-A-238: main.10's build ask, which no holder answers, is asked once more when its bound's key expires, and - // at its second bound's expiry its step ends could not run, the pipeline ends failed and frees its turn, and the finding - // call-unanswered names the ask. + // H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for + // ask-unanswered-bound (a build running four of those bounds is not touched); its one bound, + // ask-silent-bound, ends its step could not run "its ask had no end", the pipeline ending failed, its turn freed and + // the finding call-unanswered naming the ask. build := stepFor(get("main.10"), "app", seat.StepResultStepBuild) - if build.Ask == "" { + if build.Ask == "" || build.Started == nil { t.Fatalf("main.10's build = %+v, want it asked", build) } - waitFor(t, ctx, logs, func() bool { - return strings.Contains(logs.String(), "the ask "+build.Ask+" of the build step of main.10 on mesh-builds had no word from its holder within 1s: asked once more") - }) outcome("main.10", seat.PipelineOutcomeFailed) - if s := stepFor(get("main.10"), "app", seat.StepResultStepBuild); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" was not answered" { - t.Fatalf("main.10's build = %+v, want could not run, its ask not answered", s) + s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) + if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { + t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) + } + if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 { + t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask))) } waitFor(t, ctx, logs, func() bool { e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered") diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 182cb70f..1ad69414 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { t.Fatal(err) } + // mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end. + if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil { + t.Fatal(err) + } + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } // The registrations: app tests its program and tools bundle from one directory, its unit step required and its // integration step optional; web is built by node-build-node. @@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) + // H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while + // it was down is lost; a running of a module holding no silent seat puts nothing again. + firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask + publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()}) + publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()}) + waitFor(t, ctx, logs, func() bool { + buildAsksMu.Lock() + n := buildAsks[firstBuild] + buildAsksMu.Unlock() + return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1 + }) passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 58a8de3f..5ea7ce85 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -10,7 +10,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go index 3f009af3..e1d1ee1d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go @@ -1,6 +1,8 @@ package pipeline import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" @@ -16,6 +18,24 @@ import ( // from an ask until its holder's first word. const DefaultAskUnansweredBound = 10 * time.Minute +// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose +// holder says no word before the step's end (mesh-builds, mesh-wants) until that end. +const DefaultAskSilentBound = 45 * time.Minute + +// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the +// build seats, which say ask-taken (H-A-241, H-A-243). +func Silent(onSeat string) bool { return !BoundSeat(onSeat) } + +// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the +// first id and ".2", the form the holders accept, so no duplicate window is relied on. +func AgainID(id string) string { + if len(id) <= 6 { + return id + } + sum := sha256.Sum256([]byte(id + ".2")) + return id[:len(id)-6] + hex.EncodeToString(sum[:3]) +} + // The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask..` armed before the ask is put, // `ask2..` armed when it is asked once more, each with the setting as its own time to live. const ( @@ -43,11 +63,16 @@ func ParseAskBound(key string) (onSeat, ask string, again, ok bool) { // UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235). func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" } +// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)). +func SilentWhy(ask string) string { return "its ask " + ask + " had no end" } + // AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)): // Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding // call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)). type AskClock struct { + // Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat. Bound time.Duration + Silent time.Duration Arm func(key string, ttl time.Duration) error Unbound func(key string) error Raise func(onSeat, ask string, bound time.Duration) error @@ -59,12 +84,23 @@ func (c *AskClock) arm(a Asked) error { if c == nil || c.Arm == nil || a.Run != "" { return nil } - if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.bound()); err != nil { + if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil { return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err) } return nil } +// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound. +func (c *AskClock) boundOf(onSeat string) time.Duration { + if !Silent(onSeat) { + return c.bound() + } + if c.Silent > 0 { + return c.Silent + } + return DefaultAskSilentBound +} + func (c *AskClock) bound() time.Duration { if c.Bound > 0 { return c.Bound @@ -72,18 +108,21 @@ func (c *AskClock) bound() time.Duration { return DefaultAskUnansweredBound } -// Answered deletes both bounds of the ask on onSeat, its holder's first word heard: an ask-taken, a step progress or the -// step's end (H-A-238 (3)). +// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a +// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under +// AgainID goes too. func (c *AskClock) Answered(onSeat, ask string) error { if c == nil || c.Unbound == nil || ask == "" { return nil } - return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask))) + return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)), + c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask)))) } -// answeredLate clears the finding of the ask of step s, which ended unanswered, its holder heard after all (H-A-238 (2)). +// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all +// (H-A-238 (2)). func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { - if c == nil || c.Clear == nil || s.Ask == "" || s.Why != UnansweredWhy(s.Ask) { + if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) { return } if err := c.Clear(s.Seat, s.Ask); err != nil { @@ -91,10 +130,12 @@ func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { } } -// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235): at the first the ask is put once more under its -// own id, said in the journal, its second bound armed; at the second the step ends could not run "its ask was not -// answered", the finding call-unanswered raised, so the pipeline ends and its turn frees. A bound whose ask no running step -// names any longer asks nothing. +// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound, +// ask-silent-bound, ends the step could not run "its ask had no end", never asking again (that would double a running +// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its +// second bound armed under that id; the second ends the step could not run "its ask was not answered". An ended step +// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any +// longer asks nothing. func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { h, c := v.Hearer, v.Starter.Asks if c == nil { @@ -105,26 +146,30 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { return err } s := p.Steps[i] - if again { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within its two bounds: the step ends could not run (H-A-235)", - ask, s.Step, key, onSeat) - if err := h.endUnanswered(key, ask); err != nil { + if Silent(onSeat) || again { + why, within := UnansweredWhy(s.Ask), 2*c.bound() + if Silent(onSeat) { + why, within = SilentWhy(s.Ask), c.boundOf(onSeat) + } + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)", + s.Ask, s.Step, key, onSeat, within) + if err := h.endUnanswered(key, s.Ask, why); err != nil { return err } if c.Raise != nil { - return c.Raise(onSeat, ask, 2*c.bound()) + return c.Raise(onSeat, s.Ask, within) } return nil } - a, err := v.askAgain(p, s, h.now()) + a, err := v.askAgain(p, s, AgainID(s.Ask), h.now()) if err != nil { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", ask, s.Step, key, onSeat, c.bound(), err) + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err) } else { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more (H-A-235)", ask, s.Step, key, onSeat, c.bound()) + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID) } if c.Arm != nil { - if aerr := c.Arm(AskAgainBoundKey(onSeat, ask), c.bound()); aerr != nil { - return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", ask, aerr) + if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil { + return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr) } } if err != nil { @@ -134,7 +179,83 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil) } -// runningAsk is the pipeline and the index of its running step asked under ask of onSeat; an empty key when none is. +// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain +// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under +// its own id, a rerun's marks kept, said in the journal; its bound stands as armed. +func (v Verbs) HolderStarted(module string) (int, error) { + st := v.Starter + if st == nil || st.Holders == nil { + return 0, nil + } + var silent []string + for onSeat := range acceptStep { + if !Silent(onSeat) { + continue + } + holders, read := st.Holders(onSeat) + if !read { + return 0, ErrHoldersNotRead + } + if containsString(holders, module) { + silent = append(silent, onSeat) + } + } + if len(silent) == 0 { + return 0, nil + } + pkeys, err := allKeys(v.Hearer.Store) + if err != nil { + return 0, fmt.Errorf("the pipelines are not listed: %w", err) + } + now, asked := v.Hearer.now(), 0 + var first error + for _, key := range pkeys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning { + continue + } + seen := map[string]bool{} + var due []Asked + for _, s := range p.Steps { + if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) { + continue + } + seen[s.Ask] = true + a, err := v.askAgain(p, s, s.Ask, now) + if err != nil { + v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err) + continue + } + v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)", + s.Ask, s.Step, key, s.Seat, module) + due = append(due, *a) + } + if len(due) == 0 { + continue + } + st.carry(key, due) + if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil { + first = err + } + asked += len(due) + } + return asked, first +} + +// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again. +var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet") + +func containsString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty +// key when none is. func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) { pkeys, err := allKeys(v.Hearer.Store) if err != nil { @@ -152,9 +273,9 @@ func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error return "", seat.Pipeline{}, -1, nil } -// askAgain is the ask of running step s as it was put: its own id, a rerun's marks kept. -func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) { - a, err := v.placedAsk(p, s, s.Ask, now) +// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept. +func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { + a, err := v.placedAsk(p, s, id, now) if err != nil { return nil, err } @@ -183,10 +304,10 @@ func markRerun(a *Asked, caller string) { } } -// endUnanswered ends the running step of pipeline key asked under ask could not run, its ask not answered (H-A-235): every +// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every // other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when // nothing else is open, said and logged as any step's end. -func (h *Hearer) endUnanswered(key, ask string) error { +func (h *Hearer) endUnanswered(key, ask, why string) error { for { p, rev, err := h.Store.Get(key) if err != nil { @@ -198,7 +319,7 @@ func (h *Hearer) endUnanswered(key, ask string) error { } now := h.now() s := &p.Steps[i] - s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, UnansweredWhy(ask), &now + s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) var readers []int if s.Step == seat.StepResultStepValidate { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go index c32e5c2a..3d345451 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go @@ -6,8 +6,10 @@ import ( "testing" "time" + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) // H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks @@ -50,14 +52,19 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 { t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want) } - // The first bound passes: asked once more under its own id, a rerun still, the second bound armed and said. + // The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun + // still, the second bound armed under it and said (H-A-243 (3)). + again := AgainID(ask) + if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] { + t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err) + } if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil { t.Fatal(err) } - if len(asks.ids) != 2 || asks.ids[1] != ask || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, ask)+" 10m0s" { - t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, ask) + if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" { + t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again) } - if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { + if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { t.Errorf("the ask put again = %+v", asks.asks[1]) } if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) { @@ -67,7 +74,7 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { t.Fatal("the step ended at the first bound") } // The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed. - if err := v.AskUnanswered(nodebuildgo.Seat, ask, true); err != nil { + if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil { t.Fatal(err) } got, _, _ := store.Get(pkey) @@ -85,10 +92,10 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 { t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err) } - if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask)}) { + if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) { t.Errorf("a first word deleted %v (%v)", unbound, err) } - if err := v.Hearer.Taken(ask, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { + if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { t.Errorf("a late ask-taken cleared %v (%v)", cleared, err) } for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} { @@ -100,3 +107,59 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { t.Error("a drop's bound read as an ask's") } } + +// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound: +// its one bound, ask-silent-bound, ends the step could not run "its ask had no end", the finding raised. The node-engine's +// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none. +func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + var armed, raised []string + c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute, + Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil }, + Raise: func(s, ask string, within time.Duration) error { + raised = append(raised, s+" "+ask+" "+within.String()) + return nil + }} + v.Starter.Asks, v.Hearer.Asks = c, c + v.Starter.Holders = func(s string) ([]string, bool) { + return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true + } + p, rev, _ := store.Get(pkey) + // Every step has ended but app's build, asked of mesh-builds. + b := stepIndex(p, seat.StepResultStepBuild, "") + for i := range p.Steps { + if open(p.Steps[i]) && i != b { + p.Steps[i].Outcome = seat.StepResultOutcomePassed + } + } + if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) { + t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b]) + } + store.Update(pkey, p, rev) + build := p.Steps[b].Ask + if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) { + t.Fatalf("armed %v (%v), want the silent bound", armed, err) + } + // A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id. + if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 { + t.Fatalf("another holder's start put %d (%v)", n, err) + } + if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build { + t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build) + } + if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep { + t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0]) + } + // The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends. + if err := v.AskUnanswered(builds.Seat, build, false); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + s := got.Steps[b] + if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed { + t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome) + } + if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) { + t.Errorf("raised %v", raised) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 5609652d..d9d93793 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -589,7 +589,8 @@ func endOutcome(o string) bool { func stepOfAsk(p seat.Pipeline, id string) int { for i, s := range p.Steps { - if id != "" && s.Ask == id { + // An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's. + if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) { return i } } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 5f88bcf8..7de59100 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -84,7 +84,7 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", - "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "mesh-catalog.registration-changed", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", @@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool { if json.Unmarshal(raw, &m) != nil { return false } - want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index c0e1e96a..d25b519d 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -59,6 +59,7 @@ "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", + "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", @@ -158,6 +159,12 @@ "description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).", "trusted": false, "default": "10m" + }, + "ask-silent-bound": { + "type": "string", + "description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).", + "trusted": false, + "default": "45m" } }, "state": [ @@ -293,6 +300,14 @@ "mode": "0600", "content": "${config:ask-unanswered-bound}" }, + { + "id": "ask-silent-bound", + "type": "file", + "path": "${dir:state}/ask-silent-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:ask-silent-bound}" + }, { "id": "widens-read-bound", "type": "file", @@ -342,6 +357,7 @@ "MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept", "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound", "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound", + "MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound", "MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once", "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap" },