Send files over 64 KiB by digest through the bus's object bucket to nodes that read them, grant it, and warn at 75 percent (issues 1696, 1697, 1698; ADR 0486 step 3)
A node that reports file-references gets the digest and size of each file over 64 KiB that may travel so, and the content is put in the bucket declaration-files before the declaration is sent; others get every content inline. The controller alone writes the bucket and node-engines read an object by name (the four grants the node-engine's own fetch makes, no list, no write). An object goes a day after no node's latest declaration names it. The near-limit condition warns at 75 percent, and a test holds the control node's declaration, in the reference form, under half the bus's largest message.
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
|
||||
//
|
||||
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
|
||||
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
|
||||
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
|
||||
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
|
||||
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
|
||||
|
||||
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
|
||||
// no profile of reads none.
|
||||
func readsFileReferences(in compose.Input, node string) bool {
|
||||
n, ok := nodeOf(in, node)
|
||||
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
|
||||
}
|
||||
|
||||
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
|
||||
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
|
||||
// travel by digest is passed as it is.
|
||||
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
|
||||
out := make([]map[string]any, len(resources))
|
||||
var objects map[string][]byte
|
||||
for i, r := range resources {
|
||||
out[i] = r
|
||||
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
|
||||
continue
|
||||
}
|
||||
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
|
||||
content, _ := r["content"].(string)
|
||||
bytesField, _ := r["bytes"].(string)
|
||||
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
|
||||
}
|
||||
var f sdkdecl.File
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
|
||||
}
|
||||
if !f.TravelsByDigest() {
|
||||
continue
|
||||
}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(ref.Digest)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
with := make(map[string]any, len(r)+2)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
delete(with, "content")
|
||||
delete(with, "bytes")
|
||||
with["digest"], with["size"] = ref.Digest, ref.Size
|
||||
out[i] = with
|
||||
if objects == nil {
|
||||
objects = map[string][]byte{}
|
||||
}
|
||||
objects[name] = body
|
||||
}
|
||||
return out, objects, nil
|
||||
}
|
||||
|
||||
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
|
||||
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
|
||||
func namedObjects(body []byte) (map[string]int64, error) {
|
||||
var envelope struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]int64
|
||||
for _, r := range envelope.Resources {
|
||||
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
|
||||
continue
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(r.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]int64{}
|
||||
}
|
||||
out[name] = r.Size
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// objectStore is what putting the objects needs of the bucket.
|
||||
type objectStore interface {
|
||||
// Has says whether an object of that name is there, complete.
|
||||
Has(ctx context.Context, name string) (bool, error)
|
||||
// Put writes the object under that name.
|
||||
Put(ctx context.Context, name string, content []byte) error
|
||||
}
|
||||
|
||||
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a
|
||||
// node that reads the declaration finds each object. An object already there (the same content, the same name) is not
|
||||
// written again. Refuses on the first it cannot put, naming it, so the declaration is not sent.
|
||||
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
|
||||
names := make([]string, 0, len(objects))
|
||||
for name := range objects {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
have, err := store.Has(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("whether the object %s is in the bucket %s: %w", name[:12], broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
if have {
|
||||
continue
|
||||
}
|
||||
if err := store.Put(ctx, name, objects[name]); err != nil {
|
||||
return fmt.Errorf("putting the object %s (%d bytes) in the bucket %s: %w", name[:12], len(objects[name]),
|
||||
broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busObjects is the bucket on the controller's own connection.
|
||||
type busObjects struct{ store jetstream.ObjectStore }
|
||||
|
||||
func openBucket(ctx context.Context, conn *nats.Conn) (busObjects, error) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return busObjects{}, err
|
||||
}
|
||||
store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucket)
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
return busObjects{store}, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Has(ctx context.Context, name string) (bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrObjectNotFound):
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, err
|
||||
}
|
||||
return !info.Deleted, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
|
||||
_, err := b.store.PutBytes(ctx, name, content)
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user