124 lines
6.0 KiB
Go
124 lines
6.0 KiB
Go
package broker
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// The object buckets a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697): **one bucket for each
|
|
// node**, `declaration-files-<node>`.
|
|
//
|
|
// **The controller alone writes them, and a node-engine reads an object by its name from its own bucket** (ADR 0486 §2): its
|
|
// account may neither list the bucket nor write it, no account may touch another node's, and no other account has any access.
|
|
// Why a bucket each: a subject pattern cannot tell a name from a wildcard. A grant of `...M.*` matches a publish to the literal
|
|
// subject `...M.*` too (probed on nats-server 2.11.17 by the review of this issue: a node created a consumer filtered on
|
|
// `$O.<bucket>.C.*` and read every object of the bucket), so in one shared bucket a node could read every other node's files.
|
|
// With a bucket each, the same trick reads only the node's own files, which are the files of its own declaration.
|
|
//
|
|
// A bucket of the bus's object store is a stream, OBJ_<bucket>, whose messages are the objects' metadata
|
|
// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
|
|
|
|
// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
|
|
const DeclarationFilesBucket = declaration.FileBucketPrefix
|
|
|
|
// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
|
|
// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
|
|
func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
|
|
|
|
// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
|
|
func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
|
|
|
|
// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
|
|
// megabytes.
|
|
const DeclarationFilesMaxBytes = 1 << 30
|
|
|
|
// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
|
|
const DeclarationFilesKeptFor = 24 * time.Hour
|
|
|
|
// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
|
|
// chunks. The server's API for the stream is the controller's `$JS.API.>`.
|
|
func DeclarationFilesWrites(node string) (string, error) {
|
|
bucket, err := DeclarationFilesBucketOf(node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return "$O." + bucket + ".>", nil
|
|
}
|
|
|
|
// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
|
|
// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
|
|
// stream (issue 1697, notes 4 and 5):
|
|
//
|
|
// - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
|
|
// form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream.
|
|
// - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
|
|
// a consumer filtered on `$O.B.M.>` would read every object's name.
|
|
// - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
|
|
// - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
|
|
//
|
|
// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
|
|
// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
|
|
// reaches the objects of this node's own bucket and no other (above).
|
|
func DeclarationFilesReads(node string) ([]string, error) {
|
|
bucket, err := DeclarationFilesBucketOf(node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stream, err := DeclarationFilesStreamOf(node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []string{
|
|
"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
|
|
"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
|
|
"$JS.API.CONSUMER.DELETE." + stream + ".*",
|
|
"$JS.FC." + stream + ".>",
|
|
}, nil
|
|
}
|
|
|
|
// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
|
|
// bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
|
|
func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
|
|
js, err := jetstream.New(j.conn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
return EnsureDeclarationFiles(ctx, js, nodes...)
|
|
}
|
|
|
|
// EnsureDeclarationFiles is the same on a JetStream the caller holds.
|
|
func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
|
|
for _, node := range nodes {
|
|
cfg, err := DeclarationFilesConfig(node)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
|
|
return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// DeclarationFilesConfig is a node's bucket as the controller asserts it.
|
|
func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
|
|
bucket, err := DeclarationFilesBucketOf(node)
|
|
if err != nil {
|
|
return jetstream.ObjectStoreConfig{}, err
|
|
}
|
|
return jetstream.ObjectStoreConfig{
|
|
Bucket: bucket,
|
|
Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
|
|
"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
|
|
"a day after the node's latest declaration no longer names it",
|
|
MaxBytes: DeclarationFilesMaxBytes,
|
|
Storage: jetstream.FileStorage,
|
|
}, nil
|
|
}
|