A read the bus did not answer is tried again after a bound, never kept as an answer: the shadow's answer, the node's composition and the own-secret step (issue 1126)
Met twice on 2026-10-28: shanks kept 'want.shanks.brother could not be read ... no responders' as the own-secret step's last word, and ace kept 'the vault's own: not yet ... no responders' as its answer to seed-7, each until another input moved. Every bus read of the composer (manifests, builds, archives, served values, the vault's own and credentials, the have of what runs) is noted (BusFailed); an answer or a pass that met one is made again after a growing bound (5 s to 5 min), and a pass is not handed when the have of its sealed values did not answer. The own-secret step tells a read or an ask that did not answer (Retry, taken again) from a key absent or the vault's word.
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -305,6 +306,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
|
||||
t.Error("Transient misjudges a bus that does not answer, or a refusal")
|
||||
}
|
||||
// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
|
||||
if !BusFailed(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) || !BusFailed(fmt.Errorf("x: %w", nats.ErrNoResponders)) ||
|
||||
!BusFailed(fmt.Errorf("the have of web: %w", nats.ErrTimeout)) || BusFailed(fmt.Errorf("x: %w", context.Canceled)) ||
|
||||
BusFailed(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) || BusFailed(nil) {
|
||||
t.Error("BusFailed misjudges a read the bus did not answer, the run's end, or a refusal")
|
||||
}
|
||||
|
||||
// Issue 1126, C2: a config value is laid as the controller lays it (catalogue.LaidValue), from the mesh's, the node's
|
||||
// or the module's own and never from a default or an unset one, both of this node's and of what another node serves:
|
||||
@@ -427,23 +434,28 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
name, in, held string
|
||||
slots delivery.Seals
|
||||
kept bool
|
||||
fails bool // the have's read the bus did not answer: no composition, made again later (2026-10-28)
|
||||
}{
|
||||
{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true},
|
||||
{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false},
|
||||
{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true},
|
||||
{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false},
|
||||
{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false},
|
||||
{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true, false},
|
||||
{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false, false},
|
||||
{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true, false},
|
||||
{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false, false},
|
||||
{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false, false},
|
||||
{"the bus does not answer for the have", "keeper", rotatedBox, nil, false, true},
|
||||
} {
|
||||
restarted := env
|
||||
restarted.Running = func() []delivery.Composed { return nil }
|
||||
restarted.Held = func(module string) (delivery.Seals, []string) {
|
||||
restarted.Held = func(module string) (delivery.Seals, []string, error) {
|
||||
if module != row.in {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
if row.fails {
|
||||
return nil, nil, fmt.Errorf("the have of %s: %w", module, nats.ErrTimeout)
|
||||
}
|
||||
if row.slots == nil {
|
||||
return nil, []string{row.held}
|
||||
return nil, []string{row.held}, nil
|
||||
}
|
||||
return row.slots, nil
|
||||
return row.slots, nil, nil
|
||||
}
|
||||
restarted.Unseal = func(box string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
@@ -474,6 +486,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
composed = string(u.Value)
|
||||
}
|
||||
}
|
||||
if row.fails {
|
||||
if composed != "" || !again.owed.Load() {
|
||||
t.Errorf("%s: composed %v and owes a pass again %v; want nothing handed and a pass owed", row.name, composed != "", again.owed.Load())
|
||||
}
|
||||
continue
|
||||
}
|
||||
if composed == "" {
|
||||
t.Fatalf("%s: keeper was not composed", row.name)
|
||||
}
|
||||
@@ -496,11 +514,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
}
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
coreEnv.Held = func(module string) (delivery.Seals, []string) {
|
||||
coreEnv.Held = func(module string) (delivery.Seals, []string, error) {
|
||||
if module == agent.NodeModule {
|
||||
return delivery.Seals{"core-box#sealed": rotatedBox}, nil
|
||||
return delivery.Seals{"core-box#sealed": rotatedBox}, nil, nil
|
||||
}
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
if got := delivery.SealsIn(New(coreEnv).keepRunning(ownComp, nil))["core-box#sealed"]; got != rotatedBox {
|
||||
t.Errorf("the node's own core box after a restart is %s, want the one its key holds %s", got, rotatedBox)
|
||||
@@ -572,7 +590,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
|
||||
senv := readOnly.env()
|
||||
senv.Watch = sb.watch
|
||||
var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7)
|
||||
senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
|
||||
if ownFails.Load() {
|
||||
return nil, fmt.Errorf("the vault's own could not be asked: %w", nats.ErrNoResponders)
|
||||
}
|
||||
return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
|
||||
}
|
||||
// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
|
||||
@@ -780,6 +802,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
if got := answering(); got != settled+len(answers) {
|
||||
t.Errorf("what runs moved, and %d answer(s) were composed again for %d group(s)", got-settled, len(answers))
|
||||
}
|
||||
// A read the bus did not answer is no answer to keep: the group is answered again after a bound, though nothing moved.
|
||||
laterAgain = []time.Duration{20 * time.Millisecond}
|
||||
ownFails.Store(true)
|
||||
candidate("adds", 9, "lemurs", "keeper")
|
||||
waitFor("the answer met the vault not answering", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
return a["adds"].Revision == 9 && strings.Contains(a["adds"].Refused, "no responders")
|
||||
})
|
||||
ownFails.Store(false)
|
||||
waitFor("answered again once the bus answers, with nothing else moved", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
return a["adds"].Revision == 9 && a["adds"].Verdict == delivery.CandidateDiffers
|
||||
})
|
||||
drain()
|
||||
k, _ := delivery.PlacementCandidateKey("same", "n1")
|
||||
sb.del(delivery.PlacementCandidatesBucket, k)
|
||||
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 4 })
|
||||
|
||||
Reference in New Issue
Block a user