novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

A read the bus did not answer is tried again after a bound, never kept as an answer: the shadow's answer, the node's composition and the own-secret step (issue 1126)

Browse Source 
Met twice on 2026-10-28: shanks kept 'want.shanks.brother could not be read ... no
responders' as the own-secret step's last word, and ace kept 'the vault's own: not yet
... no responders' as its answer to seed-7, each until another input moved. Every bus
read of the composer (manifests, builds, archives, served values, the vault's own and
credentials, the have of what runs) is noted (BusFailed); an answer or a pass that met
one is made again after a growing bound (5 s to 5 min), and a pass is not handed when
the have of its sealed values did not answer. The own-secret step tells a read or an
ask that did not answer (Retry, taken again) from a key absent or the vault's word.
This commit is contained in:
jschoubben
2026-10-28 12:11:08 +01:00
parent 4182f8fbfb
commit cd1d2ff447
8 changed files with 306 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/selfcompose/selfcompose.go
+104 -9
View File
@@ -20,6 +20,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -66,7 +67,8 @@ type Env struct {
// before the agent applied anything, and after the switch once the kept declaration is old. By slot
// (delivery.Have.SealedAt); legacy is the list a have written before slots holds (delivery.Have.Sealed), reused
// only within its own module.
Held func(module string) (slots delivery.Seals, legacy []string)
// An error is a have the bus did not answer for, never one absent: the pass is then tried again (BusFailed).
Held func(module string) (slots delivery.Seals, legacy []string, err error)
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
Fetch func(ctx context.Context, url string) ([]byte, error)
@@ -102,6 +104,25 @@ type Composer struct {
doing string
caught bool
rev uint64
// missed counts the reads the bus did not answer (BusFailed), for a pass or an answer to tell whether it met one:
// a read that failed on the bus is tried again after a bound, never kept as an answer (issue 1126, 2026-10-28).
missed atomic.Uint64
// owed is set when a pass refused for a bus that did not answer: the composer composes again after a bound.
owed atomic.Bool
}
// note counts err when it is the bus not answering, and returns it.
func (c *Composer) note(err error) error {
if BusFailed(err) {
c.missed.Add(1)
}
return err
}
// get is env.Get, its failures noted.
func (c *Composer) get(ctx context.Context, bucket, key string) ([]byte, error) {
raw, err := c.env.Get(ctx, bucket, key)
return raw, c.note(err)
}
// New is a composer for one node.
@@ -204,11 +225,18 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
caught := make([]bool, len(ins))
ready := false
dirty := false
// A pass refused for a bus that did not answer is made again after a bound (laterAgain), not kept until another
// input moves (issue 1126, 2026-10-28).
var again <-chan time.Time
tries := 0
for {
// Drain whatever came, then compose once for all of it.
select {
case <-ctx.Done():
return
case <-again:
again = nil
dirty = true
case t := <-merged:
if !t.ok {
c.env.Say(fmt.Sprintf("the watch of %s ended; the node composes no more until its agent starts again", ins[t.in].bucket))
@@ -247,6 +275,14 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
return
}
}
if c.owed.Load() {
wait := LaterWait(tries)
tries++
c.env.Say(fmt.Sprintf("the node composes again in %s: a read on the bus did not answer", wait))
again = time.After(wait)
} else {
tries, again = 0, nil
}
if !c.caughtSent() {
select {
case out <- agent.Update{Caught: true}:
@@ -459,7 +495,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
running = c.env.Running()
}
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
missed := c.missed.Load()
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
err = c.missedIn(missed, err)
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
// tried again after a growing wait before anything is marked (issue 1126).
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
@@ -469,7 +507,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
case <-time.After(composeAgain[try]):
}
if ctx.Err() == nil {
missed = c.missed.Load()
compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
err = c.missedIn(missed, err)
}
}
if ctx.Err() != nil {
@@ -481,14 +521,25 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
c.refused(fmt.Sprintf("%v", err))
// A bus that did not answer past every try is no answer: composed again after a bound (issue 1126, 2026-10-28).
c.owed.Store(BusFailed(err))
return nil
}
c.refused("")
c.sayOnce("")
// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
missed = c.missed.Load()
for key, comp := range compositions {
compositions[key] = c.keepRunning(comp, running)
}
if c.missed.Load() > missed {
// A have of what runs the bus did not answer for: its boxes would be the vault's, sealed again, and the
// composition another; nothing is handed, and the pass is made again after a bound (issue 1126, 2026-10-28).
c.sayOnce("the node composes nothing new for now, and keeps running what it runs: the have of what it runs did not answer on the bus")
c.owed.Store(true)
return nil
}
c.owed.Store(false)
c.refused("")
c.sayOnce("")
c.mu.Lock()
defer c.mu.Unlock()
var out []agent.Update
@@ -557,7 +608,8 @@ func (c *Composer) keepRunning(comp delivery.Composed, running []delivery.Compos
kept := delivery.Seals{}
var legacy []string
if c.env.Held != nil {
slots, old := c.env.Held(module)
slots, old, err := c.env.Held(module)
c.note(err)
for slot, box := range slots {
kept[slot] = box
}
@@ -687,7 +739,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
return nil, err
}
defer r.c.waiting("the manifest " + digest + " (" + delivery.ManifestsBucket + ")")()
value, err := r.c.env.Get(ctx, delivery.ManifestsBucket, key)
value, err := r.c.get(ctx, delivery.ManifestsBucket, key)
if err != nil {
return nil, err
}
@@ -710,7 +762,7 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
return nil, err
}
defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
raw, err := r.c.env.Get(ctx, delivery.BuilderBuildsBucket, key)
raw, err := r.c.get(ctx, delivery.BuilderBuildsBucket, key)
if err != nil || raw == nil {
return nil, err
}
@@ -740,7 +792,7 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
fetching, cancel := context.WithTimeout(ctx, archiveWithin)
defer cancel()
body, err := r.c.env.Fetch(fetching, url)
if err != nil {
if r.c.note(err) != nil {
return nil, err
}
sum := sha256.Sum256(body)
@@ -759,7 +811,7 @@ func (r reader) Served(ctx context.Context, node, module string, keys []string)
defer r.c.waiting("what " + module + " serves on " + node + " (" + ResolvedBucket + ")")()
out := map[string]any{}
for _, k := range keys {
raw, err := r.c.env.Get(ctx, ResolvedBucket, node+"."+module+"."+k)
raw, err := r.c.get(ctx, ResolvedBucket, node+"."+module+"."+k)
if err != nil {
return nil, err
}
@@ -797,6 +849,7 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
}
done := r.c.waiting("the vault's own secrets")
raw, err := r.c.env.Own(ctx, map[string]any{})
r.c.note(err)
done()
if err != nil {
return nil, fmt.Errorf("the vault's own: %w", err)
@@ -853,6 +906,7 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
}
done := r.c.waiting("the vault's credentials")
raw, err := r.c.env.Own(ctx, map[string]any{"credentials": ask})
r.c.note(err)
done()
if err != nil {
return nil, nil, fmt.Errorf("the vault's own: %w", err)
@@ -938,6 +992,46 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
// composeAgain are the waits before a pass that met a bus not answering is tried again.
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
// laterAgain are the waits, after a pass or an answer refused for a bus that did not answer past composeAgain, before
// it is made again; the last repeats while the bus does not answer (issue 1126, 2026-10-28: a failed read was kept as
// the answer until another input moved).
var laterAgain = []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute}
// LaterWait is the wait before the try'th making again (from 0), of a pass, an answer or an ask the bus did not answer.
func LaterWait(try int) time.Duration { return laterAgain[min(try, len(laterAgain)-1)] }
// ErrBusMissed is a pass that met a read the bus did not answer, though the composition was made around it (a have of
// sealed values that could not be read): never handed as made.
var ErrBusMissed = errors.New("a read on the bus did not answer during the pass")
// missedIn is err, or ErrBusMissed when no error came but a read since missed failed on the bus.
func (c *Composer) missedIn(missed uint64, err error) error {
if err == nil && c.missed.Load() > missed {
return ErrBusMissed
}
return err
}
// BusFailed says err is a read the bus did not answer, for any reason but the run's own end: Transient, a read that
// timed out, or no link to the bus. Such a read is tried again after a bound, never kept as an answer; a key absent, a
// value refused or a verdict is an answer (issue 1126, 2026-10-28). Matched by value and by words, as Transient.
func BusFailed(err error) bool {
if err == nil || errors.Is(err, context.Canceled) {
return false
}
if Transient(err) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) ||
errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrDisconnected) || errors.Is(err, nats.ErrNoServers) {
return true
}
words := err.Error()
for _, w := range []string{"timeout", "deadline exceeded", "no link to the bus", "did not answer", "connection closed"} {
if strings.Contains(words, w) {
return true
}
}
return false
}
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
@@ -946,7 +1040,8 @@ func Transient(err error) bool {
if err == nil {
return false
}
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) ||
errors.Is(err, ErrBusMissed) {
return true
}
words := err.Error()