novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Take mesh-sdk at 86f9a1a5 and name each node's bucket with its FileBucketFor, refusing a node name that could carry a wildcard (issues 1696, 1697; ADR 0488)

Browse Source
This commit is contained in:
jschoubben
2026-10-21 16:05:24 +02:00
parent da364c2730
commit a3b4e0601d
15 changed files with 169 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/declaration_files.go
+7 -3
View File
@@ -159,14 +159,18 @@ func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects,
if err != nil {
return busObjects{}, err
}
store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucketOf(node))
bucket, err := broker.DeclarationFilesBucketOf(node)
if err != nil {
return busObjects{}, fmt.Errorf("the object bucket of node %q: %w", node, err)
}
store, err := js.ObjectStore(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
store, err = js.ObjectStore(ctx, broker.DeclarationFilesBucketOf(node))
store, err = js.ObjectStore(ctx, bucket)
}
}
if err != nil {
return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucketOf(node), err)
return busObjects{}, fmt.Errorf("the object bucket %s: %w", bucket, err)
}
return busObjects{store}, nil
}
cmd/mesh-controller/declaration_files_integration_test.go
+5 -1
View File
@@ -105,7 +105,11 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
if err != nil || len(named) != 3 {
t.Fatalf("what the latest declaration names: %v %v", named, err)
}
store, err := api.ObjectStore(ctx, broker.DeclarationFilesBucketOf("novox"))
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
if err != nil {
t.Fatal(err)
}
store, err := api.ObjectStore(ctx, novoxBucket)
if err != nil {
t.Fatal(err)
}
cmd/mesh-controller/declaration_sweep.go
+5 -1
View File
@@ -247,7 +247,11 @@ func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, e
if err != nil {
return filesSweep{}, err
}
store, err := js.ObjectStore(working, broker.DeclarationFilesBucketOf(node))
bucket, err := broker.DeclarationFilesBucketOf(node)
if err != nil {
return filesSweep{}, fmt.Errorf("the bucket of node %q: %w", node, err)
}
store, err := js.ObjectStore(working, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return filesSweep{}, nil // a node never sent a reference has no bucket yet
}
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021124357-863a981e014e
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021134822-86f9a1a5c508
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
go.sum
+2
View File
@@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kH
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021124357-863a981e014e h1:eq29ENikw3XfEXjxL5OdnTbavv5pu1/UnQhWWuf5V5I=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021124357-863a981e014e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021134822-86f9a1a5c508 h1:nTQN4W6gfIUR0fLpA1IYb06TGr3MtuW0xtznjbcMGLM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021134822-86f9a1a5c508/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
internal/broker/declaration_files.go
+36 -15
View File
@@ -23,14 +23,14 @@ import (
// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
const DeclarationFilesBucket = declaration.FileBucket
const DeclarationFilesBucket = declaration.FileBucketPrefix
// DeclarationFilesBucketOf is the bucket a node's files are in. One function, so the name of a node's bucket is composed in one
// place.
func DeclarationFilesBucketOf(node string) string { return DeclarationFilesBucket + "-" + node }
// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
// DeclarationFilesStreamOf is the stream under a node's bucket.
func DeclarationFilesStreamOf(node string) string { return "OBJ_" + DeclarationFilesBucketOf(node) }
// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
// megabytes.
@@ -41,7 +41,13 @@ const DeclarationFilesKeptFor = 24 * time.Hour
// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
// chunks. The server's API for the stream is the controller's `$JS.API.>`.
func DeclarationFilesWrites(node string) string { return "$O." + DeclarationFilesBucketOf(node) + ".>" }
func DeclarationFilesWrites(node string) (string, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return "", err
}
return "$O." + bucket + ".>", nil
}
// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
@@ -57,14 +63,21 @@ func DeclarationFilesWrites(node string) string { return "$O." + DeclarationFile
// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
// reaches the objects of this node's own bucket and no other (above).
func DeclarationFilesReads(node string) []string {
stream, bucket := DeclarationFilesStreamOf(node), DeclarationFilesBucketOf(node)
func DeclarationFilesReads(node string) ([]string, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return nil, err
}
stream, err := DeclarationFilesStreamOf(node)
if err != nil {
return nil, err
}
return []string{
"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
"$JS.API.CONSUMER.DELETE." + stream + ".*",
"$JS.FC." + stream + ".>",
}
}, nil
}
// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
@@ -82,21 +95,29 @@ func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
// EnsureDeclarationFiles is the same on a JetStream the caller holds.
func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
for _, node := range nodes {
if _, err := js.CreateOrUpdateObjectStore(ctx, DeclarationFilesConfig(node)); err != nil {
return fmt.Errorf("asserting the object bucket %s: %w", DeclarationFilesBucketOf(node), err)
cfg, err := DeclarationFilesConfig(node)
if err != nil {
return err
}
if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
}
}
return nil
}
// DeclarationFilesConfig is a node's bucket as the controller asserts it.
func DeclarationFilesConfig(node string) jetstream.ObjectStoreConfig {
func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return jetstream.ObjectStoreConfig{}, err
}
return jetstream.ObjectStoreConfig{
Bucket: DeclarationFilesBucketOf(node),
Bucket: bucket,
Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
"a day after the node's latest declaration no longer names it",
MaxBytes: DeclarationFilesMaxBytes,
Storage: jetstream.FileStorage,
}
}, nil
}
internal/broker/declaration_files_live_test.go
+30 -5
View File
@@ -30,7 +30,14 @@ func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, node, name string
if err != nil {
return nil, err
}
stream, bucket := DeclarationFilesStreamOf(node), DeclarationFilesBucketOf(node)
stream, err := DeclarationFilesStreamOf(node)
if err != nil {
return nil, err
}
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return nil, err
}
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
nats.DirectGet(), nats.Context(ctx))
if err != nil {
@@ -132,7 +139,7 @@ func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testi
}
put := func(node string, content []byte) string {
t.Helper()
store, err := cjs.ObjectStore(ctx, DeclarationFilesBucketOf(node))
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
if err != nil {
t.Fatal(err)
}
@@ -172,8 +179,8 @@ func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testi
}
// What each account is refused, judged by the server.
alphaStream, alphaBucket := DeclarationFilesStreamOf("alpha"), DeclarationFilesBucketOf("alpha")
betaStream, betaBucket := DeclarationFilesStreamOf("beta"), DeclarationFilesBucketOf("beta")
alphaStream, alphaBucket := mustStream(t, "alpha"), mustBucket(t, "alpha")
betaStream, betaBucket := mustStream(t, "beta"), mustBucket(t, "beta")
listing := []byte(`{"subjects_filter":">"}`)
bySeq := []byte(`{"seq":1}`)
type refusal struct {
@@ -279,7 +286,7 @@ func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testi
// Nothing was written by the refused: each bucket holds what the controller wrote.
for node, want := range map[string]int{"alpha": 2, "beta": 1} {
store, err := cjs.ObjectStore(ctx, DeclarationFilesBucketOf(node))
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
if err != nil {
t.Fatal(err)
}
@@ -289,3 +296,21 @@ func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testi
}
}
}
func mustBucket(t *testing.T, node string) string {
t.Helper()
b, err := DeclarationFilesBucketOf(node)
if err != nil {
t.Fatal(err)
}
return b
}
func mustStream(t *testing.T, node string) string {
t.Helper()
b, err := DeclarationFilesStreamOf(node)
if err != nil {
t.Fatal(err)
}
return b
}
internal/broker/declaration_files_test.go
+22
View File
@@ -107,3 +107,25 @@ func TestTheComposedUsersGiveEachNodeItsOwnBucketAndTheControllerAll(t *testing.
t.Fatalf("the controller writes the buckets of %v", controller.FilesFor)
}
}
// A node name that could carry a subject wildcard or separator into a grant is refused with an error, never composed into a bucket
// (ADR 0488): the grants of such a name would reach another node's bucket.
func TestANodeNameThatCarriesAWildcardGetsNoBucketAndNoGrant(t *testing.T) {
for _, node := range []string{"", "*", ">", "a.b", "a b", "a*"} {
if b, err := DeclarationFilesBucketOf(node); err == nil {
t.Errorf("node %q got the bucket %q", node, b)
}
if _, err := DeclarationFilesReads(node); err == nil {
t.Errorf("node %q got read grants", node)
}
if _, err := DeclarationFilesWrites(node); err == nil {
t.Errorf("node %q got write grants", node)
}
if _, err := DeclarationFilesConfig(node); err == nil {
t.Errorf("node %q got a bucket config", node)
}
}
if b, err := DeclarationFilesBucketOf("g14"); err != nil || b != "declaration-files-g14" {
t.Errorf("g14: %q %v", b, err)
}
}
internal/broker/nats.go
+10 -2
View File
@@ -743,7 +743,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
// **And the object bucket a declaration's large files travel in** (novox/hq ADR 0486 §2, issue 1697): the
// controller alone writes it, as a publish to the bucket's own subjects, which `$JS.API.>` does not cover.
for _, node := range p.FilesFor {
pub = append(pub, DeclarationFilesWrites(node))
writes, err := DeclarationFilesWrites(node)
if err != nil {
return Permissions{}, err
}
pub = append(pub, writes)
}
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
// consumer that gave up on a message, or one that was deleted. The server already publishes
@@ -846,7 +850,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
pub = append(pub, haveKeys...)
// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
// (novox/hq ADR 0486 §2, issue 1697).
pub = append(pub, DeclarationFilesReads(p.Node)...)
fileReads, err := DeclarationFilesReads(p.Node)
if err != nil {
return Permissions{}, err
}
pub = append(pub, fileReads...)
// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
if p.MovesIntoTheVault {
testdata/beside/CAPTURED
Vendored
+1 -1
View File
@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
mesh-sdk 863a981e014e08ef5781eb8767024e7754792583 conformance/events/module-event.json
mesh-sdk 86f9a1a5c5088b60f694324e84dc31f682156a1b conformance/events/module-event.json
To move them, from this repository's root, with the two repositories checked out beside it:
vendor/git.novox.be/novox/mesh-sdk/go/catalogue/resolve.go
Vendored
+18 -1
View File
@@ -904,7 +904,7 @@ func isModule(catalogue map[string]Manifest, want string) bool {
// machine can host the module as far as its capabilities go.
func missingCapabilities(m Manifest, node Node) []string {
var missing []string
for _, c := range m.Capabilities {
for _, c := range append(append([]string(nil), m.Capabilities...), impliedCapabilities(m)...) {
if !node.Capabilities[c] {
missing = append(missing, c)
}
@@ -1457,3 +1457,20 @@ func packageNames(m map[string][]string) []string {
sort.Strings(names)
return names
}
// ContainerGroups is the capability a node-engine reports when it reads a container's `groups` and passes each as a
// supplementary group (novox/hq ADR 0480 clause 7, issue 1638). An older engine refuses the key (strict decoding).
const ContainerGroups = "container-groups"
// impliedCapabilities are the capabilities a module needs for what its resources say, beyond those it declares: a
// container that names groups needs an engine that reads them, so the module is left out of a machine whose report
// lacks the capability, naming it, and the field is sent to no engine that would refuse it. None for a module whose
// containers name no groups.
func impliedCapabilities(m Manifest) []string {
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "container" && len(groupsOf(r)) > 0 {
return []string{ContainerGroups}
}
}
return nil
}
vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seat_data.go
Vendored
+2 -1
View File
@@ -66,7 +66,8 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
// names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its
// contribution out on every machine, silently, so it is refused. A detector added there is added here.
var KnownCapabilities = []string{
"battery", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", "package-manager",
"battery", "container-groups", "container-runtime", "file-references", "firewall", "graphical-session", "overlay",
"package-manager",
"power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager",
"uplink-systemd-networkd", "user-namespaces", "virtualisation",
}
vendor/git.novox.be/novox/mesh-sdk/go/declaration/declaration.go
Vendored
+1 -1
View File
@@ -326,7 +326,7 @@ type File struct {
// Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as
// the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucketFor(node), checks what it
// received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes,
// VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine
// refuses a field it does not know, whole.
vendor/git.novox.be/novox/mesh-sdk/go/declaration/filereference.go
Vendored
+28 -3
View File
@@ -14,13 +14,38 @@ const (
// FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer
// stays in the declaration as it is.
FileReferenceThreshold = 64 * 1024
// FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest
// (ObjectName). The controller alone writes it; a node-engine reads an object by name.
FileBucket = "declaration-files"
// FileBucketPrefix starts the name of a node's object bucket, FileBucketFor(node), which holds the contents of the
// files of that node's declaration, each under the hex of its sha256 digest (ObjectName). One bucket per node,
// so a grant names it and a literal `*` is never needed (ADR 0486 §2, changed by ADR 0488 §1 and §2). The controller alone writes it; the
// node's engine reads an object by name.
FileBucketPrefix = "declaration-files-"
// FileStreamPrefix starts the name of the stream behind such a bucket, which the bus's grants name.
FileStreamPrefix = "OBJ_"
// FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size.
FileReferencesCapability = "file-references"
)
var nodeToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
// FileBucketFor is the name of the object bucket of a node's files, "declaration-files-<node>" (novox/hq ADR 0488
// §4, which replaces the single bucket constant of issue 1694). The node name must be a name the bus accepts in a
// bucket and in a subject token: letters, digits, '_' and '-', so no '.', '*', '>' or space.
func FileBucketFor(node string) (string, error) {
if !nodeToken.MatchString(node) {
return "", fmt.Errorf("node %q; a node's bucket is named by letters, digits, '_' and '-' only", node)
}
return FileBucketPrefix + node, nil
}
// FileStreamFor is the name of the stream behind a node's bucket, "OBJ_" and the bucket's name.
func FileStreamFor(node string) (string, error) {
b, err := FileBucketFor(node)
if err != nil {
return "", err
}
return FileStreamPrefix + b, nil
}
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>".
vendor/modules.txt
Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021124357-863a981e014e
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021134822-86f9a1a5c508
## explicit; go 1.26.0
git.novox.be/novox/mesh-sdk/go/asks
git.novox.be/novox/mesh-sdk/go/catalogue