Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)

Live 2026-10-28: the own run's last pass, failing on its run's end, marked
every module of the controller-mode agent that replaced it (ace 60, shanks 63),
and the marks dropped the build and digest because the have read failed in the
bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx;
a pass whose context ended returns nothing; JetStream not enabled or no
responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads
the have again while the bus does not answer and marks nothing it cannot read;
baseline takes a have a composer marked as what runs, so a mark it did not make
clears; each run's shadow lives with the run.
This commit is contained in:
2026-10-28 00:34:10 +01:00
parent 199f6b48b1
commit 4993f1b1d7
9 changed files with 302 additions and 18 deletions
+44 -2
View File
@@ -452,10 +452,29 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
if c.env.Running != nil {
running = c.env.Running()
}
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
// tried again after a growing wait before anything is marked (issue 1126).
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
select {
case <-ctx.Done():
case <-time.After(composeAgain[try]):
}
if ctx.Err() == nil {
compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
}
}
if ctx.Err() != nil {
// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
return nil
}
if err != nil {
c.refused(fmt.Sprintf("%v", err))
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
c.refused(fmt.Sprintf("%v", err))
return nil
}
c.refused("")
@@ -904,3 +923,26 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
}
}
}
// composeAgain are the waits before a pass that met a bus not answering is tried again.
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
func Transient(err error) bool {
if err == nil {
return false
}
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
return true
}
words := err.Error()
for _, w := range []string{"jetstream not enabled", "no responders"} {
if strings.Contains(words, w) {
return true
}
}
return false
}
+66
View File
@@ -240,6 +240,72 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
t.Fatalf("handed %v, want %s and %s", got, lemursKey, nodeKey)
}
// Issue 1126, the switch made safe: a bus not answering for a moment (a reload of its accounts) is tried again and
// refuses nothing; a pass whose run ended refuses nothing either.
composeAgain = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
fresh := func(get func(context.Context, string, string) ([]byte, error)) (*Composer, *[]string) {
var refusals []string
e := b.env()
e.Get = get
e.Refused = func(why string) { refusals = append(refusals, why) }
c2 := New(e)
c.mu.Lock()
for k, v := range c.wants {
c2.wants[k] = v
}
for k, v := range c.placements {
c2.placements[k] = v
}
for k, v := range c.records {
c2.records[k] = v
}
for k, v := range c.modules {
c2.modules[k] = v
}
c.mu.Unlock()
return c2, &refusals
}
for name, row := range map[string]struct {
failures int // reads failing before the bus answers
cancel bool
composes bool
}{
"the bus paused for one read": {failures: 1, composes: true},
"the bus paused past every try": {failures: 100},
"the run ended mid-pass": {failures: 100, cancel: true},
} {
failed := 0
pctx, pcancel := context.WithCancel(ctx)
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
if failed < row.failures {
failed++
if row.cancel {
pcancel()
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
}
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
}
b.mu.Lock()
defer b.mu.Unlock()
return b.kv[bucket+"/"+key], nil
})
handed := c2.compose(pctx)
pcancel()
switch {
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
case row.cancel && len(*refusals) != 0:
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
}
}
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
t.Error("Transient misjudges a bus that does not answer, or a refusal")
}
// A config value of another module, and a value changing nothing composed, hand nothing.
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
quiet(t, out)