Send files over 64 KiB by digest through the bus's object bucket to nodes that read them, grant it, and warn at 75 percent (issues 1696, 1697, 1698; ADR 0486 step 3)
A node that reports file-references gets the digest and size of each file over 64 KiB that may travel so, and the content is put in the bucket declaration-files before the declaration is sent; others get every content inline. The controller alone writes the bucket and node-engines read an object by name (the four grants the node-engine's own fetch makes, no list, no write). An object goes a day after no node's latest declaration names it. The near-limit condition warns at 75 percent, and a test holds the control node's declaration, in the reference form, under half the bus's largest message.
This commit is contained in:
@@ -200,6 +200,7 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
|
||||
}
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
if !withOrder {
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
}
|
||||
@@ -281,6 +282,7 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
|
||||
if len(left) == 0 && len(in.wouldMake) == 0 {
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
here, _ := nodeOf(in.Input, node)
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
|
||||
//
|
||||
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
|
||||
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
|
||||
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
|
||||
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
|
||||
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
|
||||
|
||||
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
|
||||
// no profile of reads none.
|
||||
func readsFileReferences(in compose.Input, node string) bool {
|
||||
n, ok := nodeOf(in, node)
|
||||
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
|
||||
}
|
||||
|
||||
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
|
||||
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
|
||||
// travel by digest is passed as it is.
|
||||
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
|
||||
out := make([]map[string]any, len(resources))
|
||||
var objects map[string][]byte
|
||||
for i, r := range resources {
|
||||
out[i] = r
|
||||
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
|
||||
continue
|
||||
}
|
||||
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
|
||||
content, _ := r["content"].(string)
|
||||
bytesField, _ := r["bytes"].(string)
|
||||
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
|
||||
}
|
||||
var f sdkdecl.File
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
|
||||
}
|
||||
if !f.TravelsByDigest() {
|
||||
continue
|
||||
}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(ref.Digest)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
with := make(map[string]any, len(r)+2)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
delete(with, "content")
|
||||
delete(with, "bytes")
|
||||
with["digest"], with["size"] = ref.Digest, ref.Size
|
||||
out[i] = with
|
||||
if objects == nil {
|
||||
objects = map[string][]byte{}
|
||||
}
|
||||
objects[name] = body
|
||||
}
|
||||
return out, objects, nil
|
||||
}
|
||||
|
||||
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
|
||||
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
|
||||
func namedObjects(body []byte) (map[string]int64, error) {
|
||||
var envelope struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]int64
|
||||
for _, r := range envelope.Resources {
|
||||
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
|
||||
continue
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(r.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]int64{}
|
||||
}
|
||||
out[name] = r.Size
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// objectStore is what putting the objects needs of the bucket.
|
||||
type objectStore interface {
|
||||
// Has says whether an object of that name is there, complete.
|
||||
Has(ctx context.Context, name string) (bool, error)
|
||||
// Put writes the object under that name.
|
||||
Put(ctx context.Context, name string, content []byte) error
|
||||
}
|
||||
|
||||
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a
|
||||
// node that reads the declaration finds each object. An object already there (the same content, the same name) is not
|
||||
// written again. Refuses on the first it cannot put, naming it, so the declaration is not sent.
|
||||
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
|
||||
names := make([]string, 0, len(objects))
|
||||
for name := range objects {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
have, err := store.Has(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("whether the object %s is in the bucket %s: %w", name[:12], broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
if have {
|
||||
continue
|
||||
}
|
||||
if err := store.Put(ctx, name, objects[name]); err != nil {
|
||||
return fmt.Errorf("putting the object %s (%d bytes) in the bucket %s: %w", name[:12], len(objects[name]),
|
||||
broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busObjects is the bucket on the controller's own connection.
|
||||
type busObjects struct{ store jetstream.ObjectStore }
|
||||
|
||||
func openBucket(ctx context.Context, conn *nats.Conn) (busObjects, error) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return busObjects{}, err
|
||||
}
|
||||
store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucket)
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucket, err)
|
||||
}
|
||||
return busObjects{store}, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Has(ctx context.Context, name string) (bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrObjectNotFound):
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, err
|
||||
}
|
||||
return !info.Deleted, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
|
||||
_, err := b.store.PutBytes(ctx, name, content)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
type sigOf struct{}
|
||||
|
||||
func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
|
||||
|
||||
// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
|
||||
// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
|
||||
// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
|
||||
// fails the verification.
|
||||
func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
|
||||
MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Run twice: an assertion of an existing bucket is no error and no change.
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
|
||||
t.Fatalf("asserting the bucket again: %v", err)
|
||||
}
|
||||
|
||||
var resources []map[string]any
|
||||
for i := 0; i < 3; i++ {
|
||||
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
|
||||
}
|
||||
resources = append(resources, aFileResource("small", "tiny\n"))
|
||||
inline := sendable{Resources: resources}
|
||||
byDigest := sendable{fileReferences: true, Resources: resources}
|
||||
|
||||
bus := link.OverNATS{Conn: conn, JS: nc}
|
||||
body, err := inline.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
|
||||
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
|
||||
}
|
||||
|
||||
bucket, err := openBucket(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objects, err := byDigest.objects()
|
||||
if err != nil || len(objects) != 3 {
|
||||
t.Fatalf("objects: %d %v", len(objects), err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatalf("putting them again: %v", err)
|
||||
}
|
||||
refBody, err := byDigest.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
|
||||
t.Fatalf("the declaration by reference: %v", err)
|
||||
}
|
||||
|
||||
// What the stream holds for the node is the reference form, and it names exactly the three objects.
|
||||
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
|
||||
if err != nil || len(latest) != 1 {
|
||||
t.Fatalf("the latest declarations: %v %v", len(latest), err)
|
||||
}
|
||||
named, err := namedByLatest(latest)
|
||||
if err != nil || len(named) != 3 {
|
||||
t.Fatalf("what the latest declaration names: %v %v", named, err)
|
||||
}
|
||||
store, err := api.ObjectStore(ctx, broker.DeclarationFilesBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope struct {
|
||||
Resources []sdkdecl.File `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(refBody, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range envelope.Resources {
|
||||
if !f.IsReference() {
|
||||
continue
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(f.Digest)
|
||||
got, err := store.GetBytes(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("the object of %s: %v", f.ID, err)
|
||||
}
|
||||
if err := f.Verify(got); err != nil {
|
||||
t.Errorf("the object does not verify: %v", err)
|
||||
}
|
||||
got[len(got)/2] ^= 1
|
||||
if err := f.Verify(got); err == nil {
|
||||
t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
|
||||
// minute, and removed at the day.
|
||||
extra := bytes.Repeat([]byte("x"), 100)
|
||||
extraName := strings.Repeat("e", 64)
|
||||
if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := busSweptBucket{store}
|
||||
t0 := time.Now()
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
|
||||
t.Fatalf("first sweep: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
|
||||
t.Fatalf("a day after: %+v %v", r, err)
|
||||
}
|
||||
if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
|
||||
t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
|
||||
func contentOf(seed string, n int) string {
|
||||
var b strings.Builder
|
||||
for i := 0; b.Len() < n; i++ {
|
||||
fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
|
||||
}
|
||||
return b.String()[:n]
|
||||
}
|
||||
|
||||
func aFileResource(id, content string) map[string]any {
|
||||
return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
|
||||
}
|
||||
|
||||
// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
|
||||
// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
|
||||
// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
|
||||
// percent more of them.
|
||||
func controlNodeShape(more float64) []map[string]any {
|
||||
res := []map[string]any{
|
||||
aFileResource("nats.bus-users", contentOf("users", 252678)),
|
||||
aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
|
||||
aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
|
||||
aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
|
||||
aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
|
||||
}
|
||||
for i := 0; i < int(600*more); i++ {
|
||||
res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func wireSizeOf(t *testing.T, s sendable) int {
|
||||
t.Helper()
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The signature the controller's key makes is 64 bytes.
|
||||
msg, err := link.Envelope(body, make([]byte, 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(msg)
|
||||
}
|
||||
|
||||
// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
|
||||
// and no content, and the declaration is far under the bus's 1 MiB.
|
||||
func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
|
||||
users, kept := contentOf("users", 252678), contentOf("kept", 138475)
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
|
||||
aFileResource("small", "tiny\n"),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, want := range []string{users, kept} {
|
||||
r := got.Resources[i]
|
||||
digest := sdkdecl.DigestOf([]byte(want))
|
||||
if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
|
||||
t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
|
||||
r["digest"], r["size"], digest, len(want))
|
||||
}
|
||||
if r["path"] == nil || r["mode"] != "0644" {
|
||||
t.Errorf("%v lost its path or mode: %v", r["id"], r)
|
||||
}
|
||||
}
|
||||
if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
|
||||
t.Errorf("a small file is not inline: %v", got.Resources[2])
|
||||
}
|
||||
if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
|
||||
t.Error("the content of a file that went by digest is in the body")
|
||||
}
|
||||
if n := wireSizeOf(t, s); n > 100<<10 {
|
||||
t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
|
||||
}
|
||||
// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 2 {
|
||||
t.Fatalf("objects: %v %v", objects, err)
|
||||
}
|
||||
for _, want := range []string{users, kept} {
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
|
||||
if string(objects[name]) != want {
|
||||
t.Errorf("the object %s does not hold the content", name[:12])
|
||||
}
|
||||
}
|
||||
named, err := namedObjects(body)
|
||||
if err != nil || len(named) != 2 {
|
||||
t.Fatalf("the objects the body names: %v %v", named, err)
|
||||
}
|
||||
for name := range objects {
|
||||
if _, ok := named[name]; !ok {
|
||||
t.Errorf("the body does not name the object %s it was sent with", name[:12])
|
||||
}
|
||||
if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
|
||||
t.Errorf("the object name %q is not 64 hex characters", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
|
||||
func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
|
||||
users := contentOf("users", 252678)
|
||||
s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
|
||||
t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
|
||||
}
|
||||
if objects, err := s.objects(); err != nil || len(objects) != 0 {
|
||||
t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
|
||||
// into another file, and a directory stay inline, whatever the node reads.
|
||||
func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
|
||||
exact := contentOf("exact", 64<<10)
|
||||
big := contentOf("big", 64<<10+1)
|
||||
sealed := aFileResource("sealed", big)
|
||||
sealed["sealed"], sealed["content"] = "box", ""
|
||||
sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
|
||||
secret := aFileResource("secret", big)
|
||||
secret["secrets"] = map[string]any{"token": "x"}
|
||||
into := aFileResource("into", big)
|
||||
into["into"] = "/etc/other"
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("exact", exact), sealed, secret, into,
|
||||
{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
|
||||
aFileResource("big", big),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
|
||||
if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
|
||||
t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
|
||||
}
|
||||
}
|
||||
if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
|
||||
t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
|
||||
}
|
||||
}
|
||||
|
||||
// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
|
||||
func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
|
||||
raw := []byte(contentOf("png", 70<<10))
|
||||
r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{r}}
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 1 {
|
||||
t.Fatalf("%v %v", len(objects), err)
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
|
||||
if string(objects[name]) != string(raw) {
|
||||
t.Fatal("the object is not the decoded bytes")
|
||||
}
|
||||
if r["bytes"] == nil {
|
||||
t.Fatal("the composed resource was changed in place")
|
||||
}
|
||||
}
|
||||
|
||||
type memObjects struct {
|
||||
have map[string][]byte
|
||||
puts []string
|
||||
fail map[string]error
|
||||
}
|
||||
|
||||
func (m *memObjects) Has(_ context.Context, name string) (bool, error) {
|
||||
_, ok := m.have[name]
|
||||
return ok, nil
|
||||
}
|
||||
func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
|
||||
if err := m.fail[name]; err != nil {
|
||||
return err
|
||||
}
|
||||
m.puts = append(m.puts, name)
|
||||
m.have[name] = c
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
|
||||
func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
|
||||
objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
|
||||
b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
|
||||
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
|
||||
t.Fatalf("put %v (%v), want the one absent", b.puts, err)
|
||||
}
|
||||
b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
|
||||
err := putObjects(t.Context(), b, objects)
|
||||
if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
|
||||
t.Fatalf("a failed put: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
|
||||
var out []bucketObject
|
||||
for n, at := range b.objects {
|
||||
out = append(out, bucketObject{Name: n, UnnamedSince: at})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
|
||||
b.objects[n] = at
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
|
||||
b.objects[n] = time.Time{}
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) Delete(_ context.Context, n string) error {
|
||||
delete(b.objects, n)
|
||||
b.deleted = append(b.deleted, n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
|
||||
// only when the mark is 24 hours old; one named again loses its mark.
|
||||
func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
|
||||
named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
|
||||
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
|
||||
named3 := map[string]bool{named: true, back: true}
|
||||
r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
|
||||
if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
|
||||
t.Fatalf("marks after the first look: %v", b.objects)
|
||||
}
|
||||
// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
|
||||
r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
|
||||
r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
|
||||
t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if _, ok := b.objects[named]; !ok {
|
||||
t.Fatal("an object a latest declaration names was removed")
|
||||
}
|
||||
// An object named again before the day is up loses its mark and is not removed.
|
||||
b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
|
||||
if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
|
||||
t.Fatalf("named again: %v %v", b.objects, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
|
||||
// never "names none", which would mark an object a node needs.
|
||||
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
|
||||
body, _ := s.Body()
|
||||
msg, _ := link.Envelope(body, []byte("sig"))
|
||||
named, err := namedByLatest(map[string][]byte{"novox": msg, "ace": []byte(`{"declaration":"` + "e30=" + `","signature":""}`)})
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
|
||||
if err != nil || len(named) != 1 || !named[name] {
|
||||
t.Fatalf("%v %v", named, err)
|
||||
}
|
||||
if _, err := namedByLatest(map[string][]byte{"novox": []byte("not json")}); err == nil {
|
||||
t.Fatal("an unreadable latest declaration was read as naming none")
|
||||
}
|
||||
}
|
||||
|
||||
// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
|
||||
// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
|
||||
// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
|
||||
func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
|
||||
const limit = 1048576
|
||||
shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
|
||||
n := wireSizeOf(t, shape)
|
||||
t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
|
||||
if n > limit/2 {
|
||||
t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
|
||||
}
|
||||
if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
|
||||
t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
|
||||
}
|
||||
grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
|
||||
if grown <= limit/2 {
|
||||
t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
|
||||
}
|
||||
}
|
||||
|
||||
func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
|
||||
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
|
||||
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
|
||||
// So the size is measured at every send: a warning condition from 90% of the limit, urgent over it, cleared when
|
||||
// the declaration is back under 90%. The limit is the connection's own, not a constant.
|
||||
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
|
||||
// the declaration is back under 75%. The limit is the connection's own, not a constant.
|
||||
|
||||
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
|
||||
const kindDeclarationNearLimit = "declaration-near-limit"
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// From 90% of the bus's largest message the declaration is a warning; over it, urgent; back under 90%, cleared.
|
||||
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
|
||||
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
@@ -50,22 +50,22 @@ func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *test
|
||||
return c, found
|
||||
}
|
||||
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 899, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("a declaration at 89.9% raised a condition")
|
||||
t.Fatal("a declaration at 74.9% raised a condition")
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 900, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
|
||||
c, found := read()
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "900 bytes") ||
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
|
||||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
|
||||
t.Fatalf("at 90%% the condition is %v %+v", found, c)
|
||||
t.Fatalf("at 75%% the condition is %v %+v", found, c)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
|
||||
if c, _ := read(); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 850, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("the condition stayed after the declaration fell under 90%")
|
||||
t.Fatal("the condition stayed after the declaration fell under 75%")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
|
||||
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
|
||||
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
|
||||
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
|
||||
// found named again, that is erased; found unnamed for a day, it is removed. Objects are immutable and regenerable from the
|
||||
// controller's state, so a removal loses nothing.
|
||||
|
||||
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
|
||||
const declarationUnnamed = "unnamed-since"
|
||||
|
||||
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
|
||||
const sweepEvery = time.Hour
|
||||
|
||||
// bucketObject is one object of the bucket as the sweep sees it.
|
||||
type bucketObject struct {
|
||||
Name string
|
||||
// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
|
||||
UnnamedSince time.Time
|
||||
}
|
||||
|
||||
// sweptBucket is what the sweep needs of the bucket.
|
||||
type sweptBucket interface {
|
||||
Objects(ctx context.Context) ([]bucketObject, error)
|
||||
MarkUnnamed(ctx context.Context, name string, since time.Time) error
|
||||
ClearUnnamed(ctx context.Context, name string) error
|
||||
Delete(ctx context.Context, name string) error
|
||||
}
|
||||
|
||||
// filesSweep says what a sweep did.
|
||||
type filesSweep struct{ Kept, Marked, Cleared, Removed int }
|
||||
|
||||
// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
|
||||
// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
|
||||
func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
|
||||
var r filesSweep
|
||||
objects, err := bucket.Objects(ctx)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
|
||||
for _, o := range objects {
|
||||
switch {
|
||||
case named[o.Name]:
|
||||
r.Kept++
|
||||
if !o.UnnamedSince.IsZero() {
|
||||
if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Cleared++
|
||||
}
|
||||
case o.UnnamedSince.IsZero():
|
||||
if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Marked++
|
||||
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
|
||||
if err := bucket.Delete(ctx, o.Name); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Removed++
|
||||
default:
|
||||
r.Kept++
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// namedByLatest is the name of every object any of these latest declarations names. A body that cannot be read is an error, never
|
||||
// "names none": an object a node still needs would be marked for removal.
|
||||
func namedByLatest(bodies map[string][]byte) (map[string]bool, error) {
|
||||
named := map[string]bool{}
|
||||
for node, signed := range bodies {
|
||||
var s link.Signed
|
||||
if err := json.Unmarshal(signed, &s); err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
names, err := namedObjects(s.Declaration)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
for name := range names {
|
||||
named[name] = true
|
||||
}
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
|
||||
// no entry.
|
||||
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, node := range nodes {
|
||||
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
|
||||
}
|
||||
out[node] = msg.Data
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busSweptBucket is the bucket on the controller's connection.
|
||||
type busSweptBucket struct{ store jetstream.ObjectStore }
|
||||
|
||||
func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
|
||||
infos, err := b.store.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]bucketObject, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
o := bucketObject{Name: info.Name}
|
||||
if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
|
||||
o.UnnamedSince = at
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta := info.ObjectMeta
|
||||
kept := map[string]string{}
|
||||
for k, v := range meta.Metadata {
|
||||
kept[k] = v
|
||||
}
|
||||
change(kept)
|
||||
meta.Metadata = kept
|
||||
return b.store.UpdateMeta(ctx, name, meta)
|
||||
}
|
||||
|
||||
func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) Delete(ctx context.Context, name string) error {
|
||||
return b.store.Delete(ctx, name)
|
||||
}
|
||||
|
||||
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
|
||||
func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
tick := time.NewTicker(sweepEvery)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
|
||||
continue
|
||||
}
|
||||
why := sweepOnce(ctx, inv, conn)
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("the bucket of declaration files is swept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
|
||||
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
nodes, err := inv.Nodes(working)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the nodes cannot be read: %v", err)
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
bodies, err := latestDeclarations(working, conn, names)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
named, err := namedByLatest(bodies)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
store, err := js.ObjectStore(working, broker.DeclarationFilesBucket)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the bucket cannot be read: %v", err)
|
||||
}
|
||||
r, err := sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
|
||||
if r.Removed+r.Marked+r.Cleared > 0 {
|
||||
fmt.Printf("the bucket of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
|
||||
r.Kept, r.Marked, r.Cleared, r.Removed)
|
||||
}
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -282,6 +282,8 @@ func serve(ctx context.Context) (err error) {
|
||||
// assignment.
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
@@ -979,6 +981,18 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
|
||||
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
|
||||
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
|
||||
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
|
||||
// references, and says so; the next one tries the bucket again.
|
||||
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
|
||||
s.declared.fileReferences = false
|
||||
if body, err = s.declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err = d.declare(ctx, s, body)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1040,6 +1054,34 @@ type overTheBus struct {
|
||||
indent string
|
||||
}
|
||||
|
||||
// putFiles puts the contents s names by digest in the bucket.
|
||||
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
|
||||
objects, err := s.declared.objects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
bus, ok := b.server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
|
||||
}
|
||||
store, err := openBucket(ctx, bus.Conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if err := putObjects(putting, store, objects); err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
|
||||
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
@@ -1052,6 +1094,11 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
|
||||
// that applies it finds every one. A node reading no references has none.
|
||||
if err := b.putFiles(ctx, s); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -39,6 +39,10 @@ type sendable struct {
|
||||
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||
ReadsModules bool
|
||||
// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
|
||||
// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
|
||||
// objects). Not on the wire as a field; the bodies differ.
|
||||
fileReferences bool
|
||||
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||
moduleOf map[string]string
|
||||
@@ -115,12 +119,16 @@ type adoptionEnvelope struct {
|
||||
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
resources, _, err := s.byReference()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
envelope := map[string]any{"declaration": 1, "resources": resources}
|
||||
if s.ReadsModules {
|
||||
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||
named := make([]map[string]any, len(s.Resources))
|
||||
for n, r := range s.Resources {
|
||||
named := make([]map[string]any, len(resources))
|
||||
for n, r := range resources {
|
||||
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if !owned {
|
||||
named[n] = r
|
||||
@@ -165,3 +173,19 @@ func (s sendable) Body() ([]byte, error) {
|
||||
}
|
||||
return json.Marshal(envelope)
|
||||
}
|
||||
|
||||
// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
|
||||
// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
|
||||
func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
|
||||
if !s.fileReferences {
|
||||
return s.Resources, nil, nil
|
||||
}
|
||||
return referencing(s.Resources)
|
||||
}
|
||||
|
||||
// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
|
||||
// body is sent. Empty for a machine that is sent every content inline.
|
||||
func (s sendable) objects() (map[string][]byte, error) {
|
||||
_, objects, err := s.byReference()
|
||||
return objects, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user