Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)
Live 2026-10-28: the own run's last pass, failing on its run's end, marked every module of the controller-mode agent that replaced it (ace 60, shanks 63), and the marks dropped the build and digest because the have read failed in the bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx; a pass whose context ended returns nothing; JetStream not enabled or no responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads the have again while the bus does not answer and marks nothing it cannot read; baseline takes a have a composer marked as what runs, so a mark it did not make clears; each run's shadow lives with the run.
This commit is contained in:
@@ -452,10 +452,29 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
|
||||
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
|
||||
// tried again after a growing wait before anything is marked (issue 1126).
|
||||
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
|
||||
c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(composeAgain[try]):
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
|
||||
// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
|
||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
return nil
|
||||
}
|
||||
c.refused("")
|
||||
@@ -904,3 +923,26 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAgain are the waits before a pass that met a bus not answering is tried again.
|
||||
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
|
||||
|
||||
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
|
||||
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
|
||||
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
|
||||
// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
|
||||
func Transient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
|
||||
return true
|
||||
}
|
||||
words := err.Error()
|
||||
for _, w := range []string{"jetstream not enabled", "no responders"} {
|
||||
if strings.Contains(words, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user