Grant the self-check its ban-list question, say a refusal at once, judge the engine by its delivered version (hq to-be 45 Phase 1)
Live on 2026-10-06, two of the first self-check's findings were its own:
- D8 asked every machine's node-intrusion-prevention.banned, and the
controller's grant did not name the subject: the bus refused it 24 times
and D8 timed out after thirty seconds instead of saying so. The verbs the
self-check asks are named in broker.VerbsTheSelfCheckAsks and granted
(mesh.seat.<seat>.tool.<verb>.*); each probe declares the seat verbs it
calls, askSeatTool refuses an undeclared one, and a test over the
registry fails a probe whose question the controller is not granted.
AskSeatTool now returns a refused publish at once ("the bus refused…")
instead of waiting out its timeout; D8 asks the machines in parallel.
- D10 read every machine as behind right after a push: a node-engine says
its version as the directory it is delivered into, the archive's digest
(31045596c83a, catalogue versionOf), and D10 compared that with the
build's commit (1545b00a). It now compares with the versions the
registered build is delivered as, and a hand-placed engine's commit.
This commit is contained in:
@@ -125,6 +125,16 @@ type Principal struct {
|
||||
// goes with the retired seat row.
|
||||
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
|
||||
|
||||
// SeatVerb is one verb of one seat, on every machine holding it.
|
||||
type SeatVerb struct{ Seat, Verb string }
|
||||
|
||||
// VerbsTheSelfCheckAsks are the seat verbs the controller's self-check and watchdogs call (novox/hq
|
||||
// to-be 45 §4): D8 reads every machine's ban list. **Named one by one, and the test that holds them
|
||||
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||
var perMachineEvents = map[string]bool{"paused.*": true}
|
||||
@@ -266,6 +276,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
sub = append(sub, "mesh.seat."+ControllerSeat+".tool.>")
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
// And the verbs the self-check reads with, of any machine's holder (novox/hq to-be 45 §4).
|
||||
for _, v := range VerbsTheSelfCheckAsks {
|
||||
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
|
||||
}
|
||||
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
|
||||
// the same discovery the console reads. The question only; the answers come to its own inbox.
|
||||
pub = append(pub, "$SRV.INFO")
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -225,3 +225,59 @@ func connectionAdvisory(sub *nats.Subscription, err error) (Advisory, bool) {
|
||||
}
|
||||
return Advisory{}, false
|
||||
}
|
||||
|
||||
// Refusals of a publish, by subject, for a caller waiting on an answer to it.
|
||||
var refusalWaiters = struct {
|
||||
sync.Mutex
|
||||
hooked map[*nats.Conn]bool
|
||||
by map[string][]chan error
|
||||
}{hooked: map[*nats.Conn]bool{}, by: map[string][]chan error{}}
|
||||
|
||||
// refusalsOf is told when the bus refuses this connection a publish to subject, until stop is called.
|
||||
// The connection's error handler is chained once, before whatever it had, which still runs.
|
||||
func refusalsOf(conn *nats.Conn, subject string) (<-chan error, func()) {
|
||||
ch := make(chan error, 1)
|
||||
refusalWaiters.Lock()
|
||||
defer refusalWaiters.Unlock()
|
||||
if !refusalWaiters.hooked[conn] {
|
||||
refusalWaiters.hooked[conn] = true
|
||||
before := conn.ErrorHandler()
|
||||
conn.SetErrorHandler(func(c *nats.Conn, sub *nats.Subscription, err error) {
|
||||
if m := refusedPublish.FindStringSubmatch(errString(err)); m != nil {
|
||||
refusalWaiters.Lock()
|
||||
for _, w := range refusalWaiters.by[m[1]] {
|
||||
select {
|
||||
case w <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
refusalWaiters.Unlock()
|
||||
}
|
||||
if before != nil {
|
||||
before(c, sub, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
refusalWaiters.by[subject] = append(refusalWaiters.by[subject], ch)
|
||||
return ch, func() {
|
||||
refusalWaiters.Lock()
|
||||
defer refusalWaiters.Unlock()
|
||||
waiting := refusalWaiters.by[subject]
|
||||
for i, w := range waiting {
|
||||
if w == ch {
|
||||
refusalWaiters.by[subject] = append(waiting[:i], waiting[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(refusalWaiters.by[subject]) == 0 {
|
||||
delete(refusalWaiters.by, subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
package link
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
|
||||
// makes and deletes one, many a minute, and the bus says so each time (found running the controller
|
||||
@@ -31,3 +39,54 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// **A question the bus refuses is answered as refused at once**, not after its timeout: against a
|
||||
// server whose only user may not publish where it asks.
|
||||
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS_REFUSING")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
started := time.Now()
|
||||
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
|
||||
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
if took := time.Since(started); took > 3*time.Second {
|
||||
t.Fatalf("a refusal took %s to be known", took)
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal reaches whoever waits on that subject, and only them.
|
||||
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
|
||||
defer stop()
|
||||
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
|
||||
defer stopOther()
|
||||
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
|
||||
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
|
||||
select {
|
||||
case <-mine:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("the waiter was not told")
|
||||
}
|
||||
select {
|
||||
case <-other:
|
||||
t.Fatal("another subject's waiter was told")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
+24
-1
@@ -413,7 +413,30 @@ func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string,
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
|
||||
subject := NodeSeatToolSubject(seat, verb, node)
|
||||
// **A refused question is known at once** (novox/hq to-be 45, D8 live on 2026-10-06): the bus
|
||||
// says it to the connection and the request would otherwise wait out its whole timeout, reading
|
||||
// as a machine that did not answer.
|
||||
refused, stop := refusalsOf(conn, subject)
|
||||
defer stop()
|
||||
type replied struct {
|
||||
msg *nats.Msg
|
||||
err error
|
||||
}
|
||||
done := make(chan replied, 1)
|
||||
go func() {
|
||||
msg, err := conn.RequestWithContext(asking, subject, body)
|
||||
done <- replied{msg, err}
|
||||
}()
|
||||
var reply *nats.Msg
|
||||
select {
|
||||
case r := <-done:
|
||||
reply, err = r.msg, r.err
|
||||
case why := <-refused:
|
||||
cancel()
|
||||
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s of %s — its grants do not "+
|
||||
"name %s: %v", seat, verb, node, subject, why)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
||||
|
||||
Reference in New Issue
Block a user