Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)
Live 2026-10-28: the own run's last pass, failing on its run's end, marked every module of the controller-mode agent that replaced it (ace 60, shanks 63), and the marks dropped the build and digest because the have read failed in the bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx; a pass whose context ended returns nothing; JetStream not enabled or no responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads the have again while the bus does not answer and marks nothing it cannot read; baseline takes a have a composer marked as what runs, so a mark it did not make clears; each run's shadow lives with the run.
This commit is contained in:
@@ -452,10 +452,29 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
|
||||
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
|
||||
// tried again after a growing wait before anything is marked (issue 1126).
|
||||
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
|
||||
c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(composeAgain[try]):
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
|
||||
// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
|
||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
return nil
|
||||
}
|
||||
c.refused("")
|
||||
@@ -904,3 +923,26 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAgain are the waits before a pass that met a bus not answering is tried again.
|
||||
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
|
||||
|
||||
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
|
||||
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
|
||||
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
|
||||
// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
|
||||
func Transient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
|
||||
return true
|
||||
}
|
||||
words := err.Error()
|
||||
for _, w := range []string{"jetstream not enabled", "no responders"} {
|
||||
if strings.Contains(words, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -240,6 +240,72 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
t.Fatalf("handed %v, want %s and %s", got, lemursKey, nodeKey)
|
||||
}
|
||||
|
||||
// Issue 1126, the switch made safe: a bus not answering for a moment (a reload of its accounts) is tried again and
|
||||
// refuses nothing; a pass whose run ended refuses nothing either.
|
||||
composeAgain = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
|
||||
fresh := func(get func(context.Context, string, string) ([]byte, error)) (*Composer, *[]string) {
|
||||
var refusals []string
|
||||
e := b.env()
|
||||
e.Get = get
|
||||
e.Refused = func(why string) { refusals = append(refusals, why) }
|
||||
c2 := New(e)
|
||||
c.mu.Lock()
|
||||
for k, v := range c.wants {
|
||||
c2.wants[k] = v
|
||||
}
|
||||
for k, v := range c.placements {
|
||||
c2.placements[k] = v
|
||||
}
|
||||
for k, v := range c.records {
|
||||
c2.records[k] = v
|
||||
}
|
||||
for k, v := range c.modules {
|
||||
c2.modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return c2, &refusals
|
||||
}
|
||||
for name, row := range map[string]struct {
|
||||
failures int // reads failing before the bus answers
|
||||
cancel bool
|
||||
composes bool
|
||||
}{
|
||||
"the bus paused for one read": {failures: 1, composes: true},
|
||||
"the bus paused past every try": {failures: 100},
|
||||
"the run ended mid-pass": {failures: 100, cancel: true},
|
||||
} {
|
||||
failed := 0
|
||||
pctx, pcancel := context.WithCancel(ctx)
|
||||
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
|
||||
if failed < row.failures {
|
||||
failed++
|
||||
if row.cancel {
|
||||
pcancel()
|
||||
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
|
||||
}
|
||||
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.kv[bucket+"/"+key], nil
|
||||
})
|
||||
handed := c2.compose(pctx)
|
||||
pcancel()
|
||||
switch {
|
||||
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
|
||||
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
|
||||
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
|
||||
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
|
||||
case row.cancel && len(*refusals) != 0:
|
||||
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
|
||||
}
|
||||
}
|
||||
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
|
||||
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
|
||||
Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
|
||||
t.Error("Transient misjudges a bus that does not answer, or a refusal")
|
||||
}
|
||||
|
||||
// A config value of another module, and a value changing nothing composed, hand nothing.
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
|
||||
quiet(t, out)
|
||||
|
||||
Reference in New Issue
Block a user