A read the bus did not answer is tried again after a bound, never kept as an answer: the shadow's answer, the node's composition and the own-secret step (issue 1126)

Met twice on 2026-10-28: shanks kept 'want.shanks.brother could not be read ... no
responders' as the own-secret step's last word, and ace kept 'the vault's own: not yet
... no responders' as its answer to seed-7, each until another input moved. Every bus
read of the composer (manifests, builds, archives, served values, the vault's own and
credentials, the have of what runs) is noted (BusFailed); an answer or a pass that met
one is made again after a growing bound (5 s to 5 min), and a pass is not handed when
the have of its sealed values did not answer. The own-secret step tells a read or an
ask that did not answer (Retry, taken again) from a key absent or the vault's word.
This commit is contained in:
2026-10-28 12:11:08 +01:00
parent 4182f8fbfb
commit cd1d2ff447
8 changed files with 306 additions and 66 deletions
+46 -12
View File
@@ -9,6 +9,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -305,6 +306,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
t.Error("Transient misjudges a bus that does not answer, or a refusal")
}
// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
if !BusFailed(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) || !BusFailed(fmt.Errorf("x: %w", nats.ErrNoResponders)) ||
!BusFailed(fmt.Errorf("the have of web: %w", nats.ErrTimeout)) || BusFailed(fmt.Errorf("x: %w", context.Canceled)) ||
BusFailed(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) || BusFailed(nil) {
t.Error("BusFailed misjudges a read the bus did not answer, the run's end, or a refusal")
}
// Issue 1126, C2: a config value is laid as the controller lays it (catalogue.LaidValue), from the mesh's, the node's
// or the module's own and never from a default or an unset one, both of this node's and of what another node serves:
@@ -427,23 +434,28 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
name, in, held string
slots delivery.Seals
kept bool
fails bool // the have's read the bus did not answer: no composition, made again later (2026-10-28)
}{
{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true},
{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false},
{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true},
{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false},
{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false},
{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true, false},
{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false, false},
{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true, false},
{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false, false},
{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false, false},
{"the bus does not answer for the have", "keeper", rotatedBox, nil, false, true},
} {
restarted := env
restarted.Running = func() []delivery.Composed { return nil }
restarted.Held = func(module string) (delivery.Seals, []string) {
restarted.Held = func(module string) (delivery.Seals, []string, error) {
if module != row.in {
return nil, nil
return nil, nil, nil
}
if row.fails {
return nil, nil, fmt.Errorf("the have of %s: %w", module, nats.ErrTimeout)
}
if row.slots == nil {
return nil, []string{row.held}
return nil, []string{row.held}, nil
}
return row.slots, nil
return row.slots, nil, nil
}
restarted.Unseal = func(box string) ([]byte, error) {
b.mu.Lock()
@@ -474,6 +486,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
composed = string(u.Value)
}
}
if row.fails {
if composed != "" || !again.owed.Load() {
t.Errorf("%s: composed %v and owes a pass again %v; want nothing handed and a pass owed", row.name, composed != "", again.owed.Load())
}
continue
}
if composed == "" {
t.Fatalf("%s: keeper was not composed", row.name)
}
@@ -496,11 +514,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
}
return nil, fmt.Errorf("not sealed to n1: %s", box)
}
coreEnv.Held = func(module string) (delivery.Seals, []string) {
coreEnv.Held = func(module string) (delivery.Seals, []string, error) {
if module == agent.NodeModule {
return delivery.Seals{"core-box#sealed": rotatedBox}, nil
return delivery.Seals{"core-box#sealed": rotatedBox}, nil, nil
}
return nil, nil
return nil, nil, nil
}
if got := delivery.SealsIn(New(coreEnv).keepRunning(ownComp, nil))["core-box#sealed"]; got != rotatedBox {
t.Errorf("the node's own core box after a restart is %s, want the one its key holds %s", got, rotatedBox)
@@ -572,7 +590,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
senv := readOnly.env()
senv.Watch = sb.watch
var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7)
senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
if ownFails.Load() {
return nil, fmt.Errorf("the vault's own could not be asked: %w", nats.ErrNoResponders)
}
return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
}
// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
@@ -780,6 +802,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
if got := answering(); got != settled+len(answers) {
t.Errorf("what runs moved, and %d answer(s) were composed again for %d group(s)", got-settled, len(answers))
}
// A read the bus did not answer is no answer to keep: the group is answered again after a bound, though nothing moved.
laterAgain = []time.Duration{20 * time.Millisecond}
ownFails.Store(true)
candidate("adds", 9, "lemurs", "keeper")
waitFor("the answer met the vault not answering", func(a map[string]delivery.CandidateAnswer) bool {
return a["adds"].Revision == 9 && strings.Contains(a["adds"].Refused, "no responders")
})
ownFails.Store(false)
waitFor("answered again once the bus answers, with nothing else moved", func(a map[string]delivery.CandidateAnswer) bool {
return a["adds"].Revision == 9 && a["adds"].Verdict == delivery.CandidateDiffers
})
drain()
k, _ := delivery.PlacementCandidateKey("same", "n1")
sb.del(delivery.PlacementCandidatesBucket, k)
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 4 })