novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 2 Releases Wiki Activity

Grant the self-check its ban-list question, say a refusal at once, judge the engine by its delivered version (hq to-be 45 Phase 1)

Browse Source 
Live on 2026-10-06, two of the first self-check's findings were its own:

- D8 asked every machine's node-intrusion-prevention.banned, and the
  controller's grant did not name the subject: the bus refused it 24 times
  and D8 timed out after thirty seconds instead of saying so. The verbs the
  self-check asks are named in broker.VerbsTheSelfCheckAsks and granted
  (mesh.seat.<seat>.tool.<verb>.*); each probe declares the seat verbs it
  calls, askSeatTool refuses an undeclared one, and a test over the
  registry fails a probe whose question the controller is not granted.
  AskSeatTool now returns a refused publish at once ("the bus refused…")
  instead of waiting out its timeout; D8 asks the machines in parallel.
- D10 read every machine as behind right after a push: a node-engine says
  its version as the directory it is delivered into, the archive's digest
  (31045596c83a, catalogue versionOf), and D10 compared that with the
  build's commit (1545b00a). It now compares with the versions the
  registered build is delivered as, and a hand-placed engine's commit.
This commit is contained in:
jochen
2026-10-06 10:44:38 +02:00
parent fab6b0059e
commit 8ddc019cd2
8 changed files with 319 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/doctor.go
+25 -3
View File
@@ -61,7 +61,11 @@ type probe struct {
Phase int
// Deferred says why it is not run yet; empty for one that is.
Deferred string
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
// Asks are the seat verbs it calls. A probe may call no other (askSeatTool refuses), and the
// controller's grant names every one (a test over this registry): a probe whose question the bus
// refuses checks nothing (D8, 2026-10-06).
Asks []broker.SeatVerb
run func(ctx context.Context, d *doctor) ([]conditions.Observation, error)
}
// probeRegistry is the registry, in to-be 45's order. **The registry is the design's live form**: a
@@ -83,7 +87,8 @@ var probeRegistry = []probe{
{ID: "D7", Asserts: "every stream the controller defines exists with its definition, and its own buckets",
From: "issue 208", Kind: "stream-wrong", Phase: 1, run: probeStreams},
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans},
From: "issue 238", Kind: "own-address-banned", Phase: 1, run: probeBans,
Asks: []broker.SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}},
{ID: "D9", Asserts: "status answers in full within ten seconds, from a summary composed lately",
From: "issue 265", Kind: "status-slow", Phase: 1, run: probeStatus},
{ID: "D10", Asserts: "every machine runs the node-engine and node tools builds the mesh holds, or is inside " +
@@ -209,7 +214,7 @@ func (d *doctor) runOnce(ctx context.Context, why string) doctorRun {
wg.Add(1)
go func(i int, p probe) {
defer wg.Done()
probing, cancel := context.WithTimeout(ctx, probeWithin)
probing, cancel := context.WithTimeout(context.WithValue(ctx, probeAsksKey{}, p), probeWithin)
defer cancel()
began := time.Now()
done := make(chan result, 1)
@@ -512,3 +517,20 @@ func sortedFound(obs []conditions.Observation) []conditions.Observation {
sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
return obs
}
// probeAsksKey carries the running probe, so a seat verb it calls is checked against what it declares.
type probeAsksKey struct{}
// declaredBy says whether the probe running in ctx declared a seat verb; outside a probe, false.
func declaredBy(ctx context.Context, seat, verb string) (string, bool) {
p, ok := ctx.Value(probeAsksKey{}).(probe)
if !ok {
return "a caller outside the self-check", false
}
for _, v := range p.Asks {
if v.Seat == seat && v.Verb == verb {
return p.ID, true
}
}
return p.ID, false
}
cmd/mesh-controller/doctor_test.go
+78
View File
@@ -17,6 +17,7 @@ import (
"golang.org/x/net/dns/dnsmessage"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
@@ -345,3 +346,80 @@ func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
t.Fatalf("%v", kinds)
}
}
// **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every
// machine read as behind right after a push sent it the current build — it says the digest-named
// directory it runs from, and the mesh holds a commit).
func TestANodeEngineIsJudgedByTheVersionItIsDeliveredAs(t *testing.T) {
m := catalogue.Manifest{Module: "mesh-host", Resources: []map[string]any{
{"id": "launcher", "type": "file", "path": "/usr/lib/nox-mesh-host/launch"},
{"id": "host", "type": "archive", "path": "/usr/lib/nox-mesh-host/versions/31045596c83a"},
{"id": "unfilled", "type": "archive", "path": "/usr/lib/x/versions/${version}"},
}}
delivered := deliveredVersions(m)
if !slices.Equal(delivered, []string{"31045596c83a"}) {
t.Fatalf("%v", delivered)
}
commit := "1545b00a9f0c"
for _, c := range []struct {
reported string
behind bool
}{
{"31045596c83a", false}, // the live case: current, and was called behind
{"0123456789ab", true}, // another delivery
{"1545b00a", false}, // placed by hand, stamped with the commit
{"", false}, // not said
} {
if got := engineBehind(c.reported, delivered, commit); got != c.behind {
t.Errorf("%q behind = %v, want %v", c.reported, got, c.behind)
}
}
if engineBehind("31045596c83a", nil, commit) {
t.Error("behind a mesh that holds no delivered build")
}
}
// **Every seat verb a probe calls is one it declares, and one the controller is granted** — derived
// from the registry, so a probe added with a question the bus would refuse fails here, not live.
func TestEverySeatVerbAProbeAsksIsGranted(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
asked := 0
for _, p := range probeRegistry {
for _, v := range p.Asks {
asked++
subject := link.NodeSeatToolSubject(v.Seat, v.Verb, "anchor")
if !slices.ContainsFunc(granted.Publish, func(pattern string) bool { return subjectMatches(pattern, subject) }) {
t.Errorf("%s asks %s.%s and the controller may not publish %s", p.ID, v.Seat, v.Verb, subject)
}
if !slices.Contains(broker.VerbsTheSelfCheckAsks, v) {
t.Errorf("%s asks %s.%s, which broker.VerbsTheSelfCheckAsks does not name", p.ID, v.Seat, v.Verb)
}
}
}
if asked == 0 {
t.Fatal("no probe asks a seat verb: D8 lost its declaration")
}
// And a probe asking what it did not declare is refused before anything is sent.
ctx := context.WithValue(t.Context(), probeAsksKey{}, probe{ID: "DX"})
if _, err := askSeatTool(ctx, nil, "node-intrusion-prevention", "banned", "anchor"); err == nil ||
!strings.Contains(err.Error(), "does not declare") {
t.Fatalf("an undeclared question was asked: %v", err)
}
}
// subjectMatches is the bus's matching of a permission pattern against a subject.
func subjectMatches(pattern, subject string) bool {
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
for i, tok := range p {
if tok == ">" {
return len(s) > i
}
if i >= len(s) || (tok != "*" && tok != s[i]) {
return false
}
}
return len(p) == len(s)
}
cmd/mesh-controller/probes.go
+61 -5
View File
@@ -10,6 +10,7 @@ import (
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
@@ -576,9 +577,22 @@ func probeBans(ctx context.Context, d *doctor) ([]conditions.Observation, error)
}
sort.Strings(holders)
var out []conditions.Observation
// Every machine asked at once: one after another, four holders that each wait their bound
// outlast the probe's thirty seconds, and the probe says nothing about any of them.
answers := make([]json.RawMessage, len(holders))
errs := make([]error, len(holders))
var wg sync.WaitGroup
for i, node := range holders {
wg.Add(1)
go func(i int, node string) {
defer wg.Done()
answers[i], errs[i] = askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
}(i, node)
}
wg.Wait()
var unasked []string
for _, node := range holders {
answer, err := askSeatTool(ctx, d.js.Conn(), "node-intrusion-prevention", "banned", node)
for i, node := range holders {
answer, err := answers[i], errs[i]
if err != nil {
unasked = append(unasked, err.Error())
continue
@@ -650,6 +664,11 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
if err != nil {
return nil, err
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
hostVersions := deliveredVersions(shelf[hostModule])
rolling := map[string]bool{}
for _, p := range plans {
for m := range p.Modules {
@@ -675,9 +694,14 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
continue // a machine not heard from is S1's
}
var behind []string
host := current[hostModule].Commit
if host != "" && n.HostVersion != "" && !rolling[hostModule] && !sameCommit(n.HostVersion, host) {
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s", short(n.HostVersion), short(host)))
// **A node-engine says its version as the directory it was delivered into** — its archive's
// digest, twelve characters (catalogue.versionOf, ADR 0141) — not the commit it was built
// from. Compared as a commit, every machine read as behind right after a push sent it the
// current one (2026-10-06). An engine placed by hand reports its link-time stamp instead,
// which a commit can match.
if !rolling[hostModule] && engineBehind(n.HostVersion, hostVersions, current[hostModule].Commit) {
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
}
assigned, err := inv.Assigned(ctx, n.Name)
if err != nil {
@@ -708,6 +732,34 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
return out, nil
}
// deliveredVersions are the versions a module's registered build is delivered as: the last element
// of every resource path under a `versions/` directory, which registration filled from the artifact's
// digest (catalogue `${version}`). The node-engine names itself by that directory.
func deliveredVersions(m catalogue.Manifest) []string {
var out []string
for _, r := range m.Resources {
path, _ := r["path"].(string)
before, version, found := strings.Cut(path, "/versions/")
if !found || before == "" || version == "" || strings.Contains(version, "/") || strings.Contains(version, "$") {
continue
}
if !slices.Contains(out, version) {
out = append(out, version)
}
}
return out
}
// engineBehind says a node-engine's reported version is not the build the mesh holds: neither the
// directory that build is delivered as, nor (for an engine placed by hand) its commit. A machine that
// has not said, or a mesh that holds no delivered build, is not behind anything.
func engineBehind(reported string, delivered []string, commit string) bool {
if reported == "" || len(delivered) == 0 {
return false
}
return !slices.Contains(delivered, reported) && !sameCommit(reported, commit)
}
// hostModule is the node-engine's module.
const hostModule = "mesh-host"
@@ -750,6 +802,10 @@ func probeWatchdogs(_ context.Context, d *doctor) ([]conditions.Observation, err
// askSeatTool asks one machine's holder of a node seat a verb and answers its result; the holder's
// own refusal is an error.
func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string) (json.RawMessage, error) {
if who, declared := declaredBy(ctx, seat, verb); !declared {
return nil, fmt.Errorf("%s asks %s.%s, which it does not declare in the probe registry — and so the "+
"controller is not granted it", who, seat, verb)
}
answer, err := link.AskSeatTool(ctx, conn, seat, verb, node, map[string]any{}, 10*time.Second)
if err != nil {
return nil, err