Say when the node composes again after a seat that did not answer, say what a long pass waits on, and serve the verb compose (issue 774)

On 2026-10-31 ace's route-proxy said only that the vault could not be asked, and ace said nothing for five minutes
while a pass waited after the vault restarted; the retry that already existed was invisible. A refusal for a seat or
a read that did not answer now says when the node composes again ("the node composes again at HH:MM:SS (try N)"), a
retry pass says it starts, a pass that runs on says every minute what it waits on, and the node-engine's verb
compose makes a pass now and says how it went, so a person recovers without a restart.
This commit is contained in:
2026-10-31 00:56:47 +01:00
parent 91fb323ce0
commit 9c15610f70
7 changed files with 255 additions and 17 deletions
+57 -3
View File
@@ -6,6 +6,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
@@ -238,6 +239,20 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
}
}
b.mu.Unlock()
// Issue 774, 2026-10-31: the verb compose makes a pass now and says how it went; a composer that runs no watch
// makes none, and says so rather than waiting.
madeBefore := composed.Load()
cctx, ccancel := context.WithTimeout(ctx, 5*time.Second)
nowPass, err := c.ComposeNow(cctx)
ccancel()
if err != nil || nowPass.At.IsZero() || nowPass.Refused != "" || !nowPass.Again.IsZero() || composed.Load() == madeBefore {
t.Errorf("compose now: %+v (%v), %d pass(es) made; want one pass, composed", nowPass, err, composed.Load()-madeBefore)
}
idle, icancel := context.WithTimeout(ctx, 20*time.Millisecond)
if _, err := New(env).ComposeNow(idle); err == nil {
t.Error("a composer that watches nothing answered compose now")
}
icancel()
lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp)
nodeKey, _ := delivery.ComposedNodeKey("n1")
if !got[lemursKey] || !got[nodeKey] || len(got) != 2 {
@@ -277,15 +292,27 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
c.mu.Unlock()
return c2, &refusals
}
setFor(t, &passSaysEvery, 5*time.Millisecond)
for name, row := range map[string]struct {
failures int // reads failing before the bus answers
cancel bool
composes bool
fail error // what each failing read says; JetStream not enabled when nil
slow time.Duration // each failing read takes this long
// again: the refusal says when the node composes again, and a pass is owed (issue 774, 2026-10-31: ace's
// route-proxy said only "the vault's own could not be asked ... context deadline exceeded").
again bool
}{
"the bus paused for one read": {failures: 1, composes: true},
"the bus paused past every try": {failures: 100},
"the bus paused past every try": {failures: 100, again: true},
"the run ended mid-pass": {failures: 100, cancel: true},
"the vault did not answer in time": {failures: 100, again: true, slow: 30 * time.Millisecond,
fail: fmt.Errorf("the vault's own could not be asked: no answer from mesh.seat.mesh-vault.call.own.node~n1: %w", context.DeadlineExceeded)},
"a refusal": {failures: 100, fail: errors.New("nats says ${dir:mesh-state}, and nats declares no directory")},
} {
b.mu.Lock()
saidBefore := len(b.said)
b.mu.Unlock()
failed := 0
pctx, pcancel := context.WithCancel(ctx)
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
@@ -295,6 +322,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
pcancel()
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
}
time.Sleep(row.slow)
if row.fail != nil {
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, row.fail)
}
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
}
b.mu.Lock()
@@ -303,14 +334,37 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
})
handed := c2.compose(pctx)
pcancel()
tries := 1 + len(composeAgain) // a bus paused for a moment is tried again at once; a timeout or a refusal is not
if row.fail != nil {
tries = 1
}
switch {
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != tries):
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, tries)
case row.cancel && len(*refusals) != 0:
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
}
if !row.composes && !row.cancel && len(*refusals) > 0 {
says := strings.Contains((*refusals)[0], "; the node composes again at ") && strings.HasSuffix((*refusals)[0], "(try 1)")
if says != row.again || c2.owed.Load() != row.again || (c2.last.Again.IsZero() == row.again) || c2.last.Refused != (*refusals)[0] {
t.Errorf("%s: refused %q, owes a pass %v, last pass %+v; want the time it composes again said and owed: %v",
name, (*refusals)[0], c2.owed.Load(), c2.last, row.again)
}
}
// A pass that runs on says so, and what it waits on (issue 774, 2026-10-31: ace was silent for five minutes).
if row.slow > 0 {
b.mu.Lock()
long := false
for _, line := range b.said[saidBefore:] {
long = long || strings.Contains(line, "the node's composition has run for ") && strings.Contains(line, " and waits on ")
}
b.mu.Unlock()
if !long {
t.Errorf("%s: a pass that ran on said nothing of what it waits on", name)
}
}
}
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||