Retry an apply, a fetch by digest and a have write that met the bus reloading, and keep the cause in the have
mesh/delivery merged: source.main moved to novox/mesh-host@1195cf45971e
mesh/check pass 99285cb080db on 1195cf45971e | unit 303 run, 1080 kept, 5 skipped | integration 306 run, 1082 kept, 4 skipped | over bound: unit (slow…
mesh/delivery merged: source.main moved to novox/mesh-host@1195cf45971e
mesh/check pass 99285cb080db on 1195cf45971e | unit 303 run, 1080 kept, 5 skipped | integration 306 run, 1082 kept, 4 skipped | over bound: unit (slow…
The bus node's engine writes the bus user list (ADR 0547 bridge); each write makes the bus reload its accounts, and for about three seconds every JetStream request meets no responders. An apply that failed then was kept failed until a key changed (mesh-vault on novox, 2026-10-25 23:04:24), holding every module that needs the secret seat. A transient bus failure is now retried with a backoff (2 s, 5 s, 15 s, 60 s, then every 5 min), the have saying retrying meanwhile; after the engine writes the user list, the transiently failed modules are tried again once the reload is over. A have write that met it is written again the same way. shorten keeps the head and the tail, the cause.
This commit is contained in:
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -9,13 +10,20 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// aFilesLink is a link that serves objects by name.
|
||||
type aFilesLink struct{ objects map[string][]byte }
|
||||
// aFilesLink is a link that serves objects by name, or fails every read with err.
|
||||
type aFilesLink struct {
|
||||
objects map[string][]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
|
||||
if a.err != nil {
|
||||
return nil, a.err
|
||||
}
|
||||
if b, ok := a.objects[name]; ok {
|
||||
return b, nil
|
||||
}
|
||||
@@ -40,7 +48,7 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
||||
|
||||
bad := append([]byte(nil), body...)
|
||||
bad[10] ^= 1
|
||||
ctx := link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: bad}})
|
||||
ctx := link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: bad}})
|
||||
report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
|
||||
t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
|
||||
@@ -48,8 +56,24 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
t.Error("1695/AC2: a file was written from an object that differs")
|
||||
}
|
||||
if failedWith(report.Refused).Transient {
|
||||
t.Errorf("an object that differs is a real failure, and was taken as the bus not answering: %s", report.Refused)
|
||||
}
|
||||
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: body}})
|
||||
// The bus answers no JetStream request while it reloads its accounts (2026-10-25 23:04:24, mesh-vault on the bus
|
||||
// node): the failure is transient, so the agent tries it again, and the detail kept is its cause, the tail.
|
||||
noResponders := fmt.Errorf("fetching the object 2bd98ebd of this node's own object bucket: reading the chunks of "+
|
||||
"object %s: %w", strings.Repeat("x", 300), nats.ErrNoResponders)
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{err: noResponders})
|
||||
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
failed := failedWith("a file of this composition could not be had by its digest: file mesh-vault.kept: " + report.Refused)
|
||||
if !failed.Transient || !strings.HasSuffix(failed.Detail, nats.ErrNoResponders.Error()) || len(failed.Detail) > 310 ||
|
||||
!strings.HasPrefix(failed.Detail, "a file of this composition") {
|
||||
t.Errorf("a fetch that met no responders: transient %v, detail %q (%d bytes); want transient, its head and its cause",
|
||||
failed.Transient, failed.Detail, len(failed.Detail))
|
||||
}
|
||||
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: body}})
|
||||
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("1695/AC1: refused: %s", report.Refused)
|
||||
|
||||
Reference in New Issue
Block a user