Retry an apply, a fetch by digest and a have write that met the bus reloading, and keep the cause in the have
mesh/delivery merged: source.main moved to novox/mesh-host@1195cf45971e
mesh/check pass 99285cb080db on 1195cf45971e | unit 303 run, 1080 kept, 5 skipped | integration 306 run, 1082 kept, 4 skipped | over bound: unit (slow…

The bus node's engine writes the bus user list (ADR 0547 bridge); each write makes the bus reload its accounts, and
for about three seconds every JetStream request meets no responders. An apply that failed then was kept failed until a
key changed (mesh-vault on novox, 2026-10-25 23:04:24), holding every module that needs the secret seat. A transient
bus failure is now retried with a backoff (2 s, 5 s, 15 s, 60 s, then every 5 min), the have saying retrying meanwhile;
after the engine writes the user list, the transiently failed modules are tried again once the reload is over. A have
write that met it is written again the same way. shorten keeps the head and the tail, the cause.
This commit is contained in:
2026-10-25 23:37:02 +01:00
parent d75c66dc7e
commit 99285cb080
9 changed files with 390 additions and 57 deletions
+28 -4
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
@@ -9,13 +10,20 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-host/internal/link"
)
// aFilesLink is a link that serves objects by name.
type aFilesLink struct{ objects map[string][]byte }
// aFilesLink is a link that serves objects by name, or fails every read with err.
type aFilesLink struct {
objects map[string][]byte
err error
}
func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
if a.err != nil {
return nil, a.err
}
if b, ok := a.objects[name]; ok {
return b, nil
}
@@ -40,7 +48,7 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
bad := append([]byte(nil), body...)
bad[10] ^= 1
ctx := link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: bad}})
ctx := link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: bad}})
report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
@@ -48,8 +56,24 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
if _, err := os.Stat(target); err == nil {
t.Error("1695/AC2: a file was written from an object that differs")
}
if failedWith(report.Refused).Transient {
t.Errorf("an object that differs is a real failure, and was taken as the bus not answering: %s", report.Refused)
}
ctx = link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: body}})
// The bus answers no JetStream request while it reloads its accounts (2026-10-25 23:04:24, mesh-vault on the bus
// node): the failure is transient, so the agent tries it again, and the detail kept is its cause, the tail.
noResponders := fmt.Errorf("fetching the object 2bd98ebd of this node's own object bucket: reading the chunks of "+
"object %s: %w", strings.Repeat("x", 300), nats.ErrNoResponders)
ctx = link.WithFiles(context.Background(), aFilesLink{err: noResponders})
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
failed := failedWith("a file of this composition could not be had by its digest: file mesh-vault.kept: " + report.Refused)
if !failed.Transient || !strings.HasSuffix(failed.Detail, nats.ErrNoResponders.Error()) || len(failed.Detail) > 310 ||
!strings.HasPrefix(failed.Detail, "a file of this composition") {
t.Errorf("a fetch that met no responders: transient %v, detail %q (%d bytes); want transient, its head and its cause",
failed.Transient, failed.Detail, len(failed.Detail))
}
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: body}})
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
if report.Refused != "" {
t.Fatalf("1695/AC1: refused: %s", report.Refused)