Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)

Live 2026-10-28: the own run's last pass, failing on its run's end, marked
every module of the controller-mode agent that replaced it (ace 60, shanks 63),
and the marks dropped the build and digest because the have read failed in the
bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx;
a pass whose context ended returns nothing; JetStream not enabled or no
responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads
the have again while the bus does not answer and marks nothing it cannot read;
baseline takes a have a composer marked as what runs, so a mark it did not make
clears; each run's shadow lives with the run.
This commit is contained in:
2026-10-28 00:34:10 +01:00
parent 199f6b48b1
commit 4993f1b1d7
9 changed files with 302 additions and 18 deletions
+44 -2
View File
@@ -452,10 +452,29 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
if c.env.Running != nil {
running = c.env.Running()
}
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
// tried again after a growing wait before anything is marked (issue 1126).
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
select {
case <-ctx.Done():
case <-time.After(composeAgain[try]):
}
if ctx.Err() == nil {
compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
}
}
if ctx.Err() != nil {
// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
return nil
}
if err != nil {
c.refused(fmt.Sprintf("%v", err))
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
c.refused(fmt.Sprintf("%v", err))
return nil
}
c.refused("")
@@ -904,3 +923,26 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
}
}
}
// composeAgain are the waits before a pass that met a bus not answering is tried again.
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
func Transient(err error) bool {
if err == nil {
return false
}
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
return true
}
words := err.Error()
for _, w := range []string{"jetstream not enabled", "no responders"} {
if strings.Contains(words, w) {
return true
}
}
return false
}