Grant the self-check its ban-list question, say a refusal at once, judge the engine by its delivered version (hq to-be 45 Phase 1)

Live on 2026-10-06, two of the first self-check's findings were its own:

- D8 asked every machine's node-intrusion-prevention.banned, and the
  controller's grant did not name the subject: the bus refused it 24 times
  and D8 timed out after thirty seconds instead of saying so. The verbs the
  self-check asks are named in broker.VerbsTheSelfCheckAsks and granted
  (mesh.seat.<seat>.tool.<verb>.*); each probe declares the seat verbs it
  calls, askSeatTool refuses an undeclared one, and a test over the
  registry fails a probe whose question the controller is not granted.
  AskSeatTool now returns a refused publish at once ("the bus refused…")
  instead of waiting out its timeout; D8 asks the machines in parallel.
- D10 read every machine as behind right after a push: a node-engine says
  its version as the directory it is delivered into, the archive's digest
  (31045596c83a, catalogue versionOf), and D10 compared that with the
  build's commit (1545b00a). It now compares with the versions the
  registered build is delivered as, and a hand-placed engine's commit.
This commit is contained in:
jochen
2026-10-06 10:44:38 +02:00
parent fab6b0059e
commit 8ddc019cd2
8 changed files with 319 additions and 11 deletions
+56
View File
@@ -225,3 +225,59 @@ func connectionAdvisory(sub *nats.Subscription, err error) (Advisory, bool) {
}
return Advisory{}, false
}
// Refusals of a publish, by subject, for a caller waiting on an answer to it.
var refusalWaiters = struct {
sync.Mutex
hooked map[*nats.Conn]bool
by map[string][]chan error
}{hooked: map[*nats.Conn]bool{}, by: map[string][]chan error{}}
// refusalsOf is told when the bus refuses this connection a publish to subject, until stop is called.
// The connection's error handler is chained once, before whatever it had, which still runs.
func refusalsOf(conn *nats.Conn, subject string) (<-chan error, func()) {
ch := make(chan error, 1)
refusalWaiters.Lock()
defer refusalWaiters.Unlock()
if !refusalWaiters.hooked[conn] {
refusalWaiters.hooked[conn] = true
before := conn.ErrorHandler()
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
if m := refusedPublish.FindStringSubmatch(errString(err)); m != nil {
refusalWaiters.Lock()
for _, w := range refusalWaiters.by[m[1]] {
select {
case w <- err:
default:
}
}
refusalWaiters.Unlock()
}
if before != nil {
before(c, sub, err)
}
})
}
refusalWaiters.by[subject] = append(refusalWaiters.by[subject], ch)
return ch, func() {
refusalWaiters.Lock()
defer refusalWaiters.Unlock()
waiting := refusalWaiters.by[subject]
for i, w := range waiting {
if w == ch {
refusalWaiters.by[subject] = append(waiting[:i], waiting[i+1:]...)
break
}
}
if len(refusalWaiters.by[subject]) == 0 {
delete(refusalWaiters.by, subject)
}
}
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
+60 -1
View File
@@ -1,6 +1,14 @@
package link
import "testing"
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
// makes and deletes one, many a minute, and the bus says so each time (found running the controller
@@ -31,3 +39,54 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
t.Fatalf("%+v", a)
}
}
// **A question the bus refuses is answered as refused at once**, not after its timeout: against a
// server whose only user may not publish where it asks.
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_REFUSING")
if url == "" {
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
started := time.Now()
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
t.Fatalf("answered %v", err)
}
if took := time.Since(started); took > 3*time.Second {
t.Fatalf("a refusal took %s to be known", took)
}
}
// The refusal reaches whoever waits on that subject, and only them.
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
defer stop()
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
defer stopOther()
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
select {
case <-mine:
case <-time.After(time.Second):
t.Fatal("the waiter was not told")
}
select {
case <-other:
t.Fatal("another subject's waiter was told")
default:
}
}
+24 -1
View File
@@ -413,7 +413,30 @@ func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string,
}
asking, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
subject := NodeSeatToolSubject(seat, verb, node)
// **A refused question is known at once** (novox/hq to-be 45, D8 live on 2026-10-06): the bus
// says it to the connection and the request would otherwise wait out its whole timeout, reading
// as a machine that did not answer.
refused, stop := refusalsOf(conn, subject)
defer stop()
type replied struct {
msg *nats.Msg
err error
}
done := make(chan replied, 1)
go func() {
msg, err := conn.RequestWithContext(asking, subject, body)
done <- replied{msg, err}
}()
var reply *nats.Msg
select {
case r := <-done:
reply, err = r.msg, r.err
case why := <-refused:
cancel()
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s of %s — its grants do not "+
"name %s: %v", seat, verb, node, subject, why)
}
switch {
case errors.Is(err, nats.ErrNoResponders):
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+