Send files over 64 KiB by digest through the bus's object bucket to nodes that read them, grant it, and warn at 75 percent (issues 1696, 1697, 1698; ADR 0486 step 3)

A node that reports file-references gets the digest and size of each file over 64 KiB that may travel so, and the content is put in
the bucket declaration-files before the declaration is sent; others get every content inline. The controller alone writes the
bucket and node-engines read an object by name (the four grants the node-engine's own fetch makes, no list, no write). An object
goes a day after no node's latest declaration names it. The near-limit condition warns at 75 percent, and a test holds the control
node's declaration, in the reference form, under half the bus's largest message.
This commit is contained in:
2026-10-21 15:16:11 +02:00
parent fa5b5e1c11
commit ceaa616d01
18 changed files with 1486 additions and 18 deletions
+2
View File
@@ -200,6 +200,7 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
}
sayFound(ctx, d)
out := sendableOf(d)
out.fileReferences = readsFileReferences(in.Input, node)
if !withOrder {
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
}
@@ -281,6 +282,7 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
if len(left) == 0 && len(in.wouldMake) == 0 {
sayFound(ctx, d)
out := sendableOf(d)
out.fileReferences = readsFileReferences(in.Input, node)
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
here, _ := nodeOf(in.Input, node)
+177
View File
@@ -0,0 +1,177 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"git.novox.be/novox/mesh-sdk/go/compose"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
//
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
// no profile of reads none.
func readsFileReferences(in compose.Input, node string) bool {
n, ok := nodeOf(in, node)
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
}
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
// travel by digest is passed as it is.
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
out := make([]map[string]any, len(resources))
var objects map[string][]byte
for i, r := range resources {
out[i] = r
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
continue
}
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
content, _ := r["content"].(string)
bytesField, _ := r["bytes"].(string)
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
continue
}
raw, err := json.Marshal(r)
if err != nil {
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
}
var f sdkdecl.File
if err := json.Unmarshal(raw, &f); err != nil {
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
}
if !f.TravelsByDigest() {
continue
}
ref, body, err := f.AsReference()
if err != nil {
return nil, nil, err
}
name, err := sdkdecl.ObjectName(ref.Digest)
if err != nil {
return nil, nil, err
}
with := make(map[string]any, len(r)+2)
for k, v := range r {
with[k] = v
}
delete(with, "content")
delete(with, "bytes")
with["digest"], with["size"] = ref.Digest, ref.Size
out[i] = with
if objects == nil {
objects = map[string][]byte{}
}
objects[name] = body
}
return out, objects, nil
}
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
func namedObjects(body []byte) (map[string]int64, error) {
var envelope struct {
Resources []struct {
Type string `json:"type"`
Digest string `json:"digest"`
Size int64 `json:"size"`
} `json:"resources"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
return nil, err
}
var out map[string]int64
for _, r := range envelope.Resources {
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
continue
}
name, err := sdkdecl.ObjectName(r.Digest)
if err != nil {
return nil, err
}
if out == nil {
out = map[string]int64{}
}
out[name] = r.Size
}
return out, nil
}
// objectStore is what putting the objects needs of the bucket.
type objectStore interface {
// Has says whether an object of that name is there, complete.
Has(ctx context.Context, name string) (bool, error)
// Put writes the object under that name.
Put(ctx context.Context, name string, content []byte) error
}
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a
// node that reads the declaration finds each object. An object already there (the same content, the same name) is not
// written again. Refuses on the first it cannot put, naming it, so the declaration is not sent.
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
names := make([]string, 0, len(objects))
for name := range objects {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
have, err := store.Has(ctx, name)
if err != nil {
return fmt.Errorf("whether the object %s is in the bucket %s: %w", name[:12], broker.DeclarationFilesBucket, err)
}
if have {
continue
}
if err := store.Put(ctx, name, objects[name]); err != nil {
return fmt.Errorf("putting the object %s (%d bytes) in the bucket %s: %w", name[:12], len(objects[name]),
broker.DeclarationFilesBucket, err)
}
}
return nil
}
// busObjects is the bucket on the controller's own connection.
type busObjects struct{ store jetstream.ObjectStore }
func openBucket(ctx context.Context, conn *nats.Conn) (busObjects, error) {
js, err := jetstream.New(conn)
if err != nil {
return busObjects{}, err
}
store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucket)
if err != nil {
return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucket, err)
}
return busObjects{store}, nil
}
func (b busObjects) Has(ctx context.Context, name string) (bool, error) {
info, err := b.store.GetInfo(ctx, name)
switch {
case errors.Is(err, jetstream.ErrObjectNotFound):
return false, nil
case err != nil:
return false, err
}
return !info.Deleted, nil
}
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
_, err := b.store.PutBytes(ctx, name, content)
return err
}
@@ -0,0 +1,157 @@
//go:build integration
package main
import (
"bytes"
"context"
"encoding/json"
"strings"
"testing"
"time"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
type sigOf struct{}
func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
// fails the verification.
func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
s := testbus.Start(t)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
nc, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
api, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
t.Fatal(err)
}
// Run twice: an assertion of an existing bucket is no error and no change.
if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
t.Fatalf("asserting the bucket again: %v", err)
}
var resources []map[string]any
for i := 0; i < 3; i++ {
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
}
resources = append(resources, aFileResource("small", "tiny\n"))
inline := sendable{Resources: resources}
byDigest := sendable{fileReferences: true, Resources: resources}
bus := link.OverNATS{Conn: conn, JS: nc}
body, err := inline.Body()
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
}
bucket, err := openBucket(ctx, conn)
if err != nil {
t.Fatal(err)
}
objects, err := byDigest.objects()
if err != nil || len(objects) != 3 {
t.Fatalf("objects: %d %v", len(objects), err)
}
if err := putObjects(ctx, bucket, objects); err != nil {
t.Fatal(err)
}
if err := putObjects(ctx, bucket, objects); err != nil {
t.Fatalf("putting them again: %v", err)
}
refBody, err := byDigest.Body()
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
t.Fatalf("the declaration by reference: %v", err)
}
// What the stream holds for the node is the reference form, and it names exactly the three objects.
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
if err != nil || len(latest) != 1 {
t.Fatalf("the latest declarations: %v %v", len(latest), err)
}
named, err := namedByLatest(latest)
if err != nil || len(named) != 3 {
t.Fatalf("what the latest declaration names: %v %v", named, err)
}
store, err := api.ObjectStore(ctx, broker.DeclarationFilesBucket)
if err != nil {
t.Fatal(err)
}
var envelope struct {
Resources []sdkdecl.File `json:"resources"`
}
if err := json.Unmarshal(refBody, &envelope); err != nil {
t.Fatal(err)
}
for _, f := range envelope.Resources {
if !f.IsReference() {
continue
}
name, _ := sdkdecl.ObjectName(f.Digest)
got, err := store.GetBytes(ctx, name)
if err != nil {
t.Fatalf("the object of %s: %v", f.ID, err)
}
if err := f.Verify(got); err != nil {
t.Errorf("the object does not verify: %v", err)
}
got[len(got)/2] ^= 1
if err := f.Verify(got); err == nil {
t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
}
}
// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
// minute, and removed at the day.
extra := bytes.Repeat([]byte("x"), 100)
extraName := strings.Repeat("e", 64)
if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
t.Fatal(err)
}
b := busSweptBucket{store}
t0 := time.Now()
if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
t.Fatalf("first sweep: %+v %v", r, err)
}
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
t.Fatalf("a minute short of a day: %+v %v", r, err)
}
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
t.Fatalf("a day after: %+v %v", r, err)
}
if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
}
}
@@ -0,0 +1,381 @@
package main
import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-controller/internal/link"
)
// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
func contentOf(seed string, n int) string {
var b strings.Builder
for i := 0; b.Len() < n; i++ {
fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
}
return b.String()[:n]
}
func aFileResource(id, content string) map[string]any {
return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
}
// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
// percent more of them.
func controlNodeShape(more float64) []map[string]any {
res := []map[string]any{
aFileResource("nats.bus-users", contentOf("users", 252678)),
aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
}
for i := 0; i < int(600*more); i++ {
res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
}
return res
}
func wireSizeOf(t *testing.T, s sendable) int {
t.Helper()
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
// The signature the controller's key makes is 64 bytes.
msg, err := link.Envelope(body, make([]byte, 64))
if err != nil {
t.Fatal(err)
}
return len(msg)
}
// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
// and no content, and the declaration is far under the bus's 1 MiB.
func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
users, kept := contentOf("users", 252678), contentOf("kept", 138475)
s := sendable{fileReferences: true, Resources: []map[string]any{
aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
aFileResource("small", "tiny\n"),
}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
for i, want := range []string{users, kept} {
r := got.Resources[i]
digest := sdkdecl.DigestOf([]byte(want))
if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
r["digest"], r["size"], digest, len(want))
}
if r["path"] == nil || r["mode"] != "0644" {
t.Errorf("%v lost its path or mode: %v", r["id"], r)
}
}
if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
t.Errorf("a small file is not inline: %v", got.Resources[2])
}
if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
t.Error("the content of a file that went by digest is in the body")
}
if n := wireSizeOf(t, s); n > 100<<10 {
t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
}
// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
objects, err := s.objects()
if err != nil || len(objects) != 2 {
t.Fatalf("objects: %v %v", objects, err)
}
for _, want := range []string{users, kept} {
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
if string(objects[name]) != want {
t.Errorf("the object %s does not hold the content", name[:12])
}
}
named, err := namedObjects(body)
if err != nil || len(named) != 2 {
t.Fatalf("the objects the body names: %v %v", named, err)
}
for name := range objects {
if _, ok := named[name]; !ok {
t.Errorf("the body does not name the object %s it was sent with", name[:12])
}
if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
t.Errorf("the object name %q is not 64 hex characters", name)
}
}
}
// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
users := contentOf("users", 252678)
s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
}
if objects, err := s.objects(); err != nil || len(objects) != 0 {
t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
}
}
// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
// into another file, and a directory stay inline, whatever the node reads.
func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
exact := contentOf("exact", 64<<10)
big := contentOf("big", 64<<10+1)
sealed := aFileResource("sealed", big)
sealed["sealed"], sealed["content"] = "box", ""
sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
secret := aFileResource("secret", big)
secret["secrets"] = map[string]any{"token": "x"}
into := aFileResource("into", big)
into["into"] = "/etc/other"
s := sendable{fileReferences: true, Resources: []map[string]any{
aFileResource("exact", exact), sealed, secret, into,
{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
aFileResource("big", big),
}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
}
}
if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
}
}
// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
raw := []byte(contentOf("png", 70<<10))
r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
s := sendable{fileReferences: true, Resources: []map[string]any{r}}
objects, err := s.objects()
if err != nil || len(objects) != 1 {
t.Fatalf("%v %v", len(objects), err)
}
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
if string(objects[name]) != string(raw) {
t.Fatal("the object is not the decoded bytes")
}
if r["bytes"] == nil {
t.Fatal("the composed resource was changed in place")
}
}
type memObjects struct {
have map[string][]byte
puts []string
fail map[string]error
}
func (m *memObjects) Has(_ context.Context, name string) (bool, error) {
_, ok := m.have[name]
return ok, nil
}
func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
if err := m.fail[name]; err != nil {
return err
}
m.puts = append(m.puts, name)
m.have[name] = c
return nil
}
// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
t.Fatalf("put %v (%v), want the one absent", b.puts, err)
}
b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
err := putObjects(t.Context(), b, objects)
if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
t.Fatalf("a failed put: %v", err)
}
}
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
// controller's grant does, and takes the second.
type fallbackDelivery struct {
bodies [][]byte
first error
}
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
f.bodies = append(f.bodies, body)
if len(f.bodies) == 1 && f.first != nil {
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
}
return "digest", nil
}
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
// with every content inline and is made; a send that fails for another reason is not retried inline.
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
users := contentOf("users", 100<<10)
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
f := &fallbackDelivery{first: errors.New("timeout")}
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
}
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
t.Error("the first body should name the file by digest and the second carry it inline")
}
other := &fallbackDelivery{}
other.first = nil
failing := &failingDelivery{err: errors.New("the bus is down")}
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
}
}
type failingDelivery struct {
err error
calls int
}
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
f.calls++
return "", f.err
}
// A bucket for the sweep.
type sweepBucketFake struct {
objects map[string]time.Time // zero: named at the last look
deleted []string
}
func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
var out []bucketObject
for n, at := range b.objects {
out = append(out, bucketObject{Name: n, UnnamedSince: at})
}
return out, nil
}
func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
b.objects[n] = at
return nil
}
func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
b.objects[n] = time.Time{}
return nil
}
func (b *sweepBucketFake) Delete(_ context.Context, n string) error {
delete(b.objects, n)
b.deleted = append(b.deleted, n)
return nil
}
// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
// only when the mark is 24 hours old; one named again loses its mark.
func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
named3 := map[string]bool{named: true, back: true}
r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
}
if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
t.Fatalf("marks after the first look: %v", b.objects)
}
// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
r.Removed != 0 || len(b.deleted) != 0 {
t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
}
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
}
if _, ok := b.objects[named]; !ok {
t.Fatal("an object a latest declaration names was removed")
}
// An object named again before the day is up loses its mark and is not removed.
b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
t.Fatalf("named again: %v %v", b.objects, err)
}
}
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
// never "names none", which would mark an object a node needs.
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
body, _ := s.Body()
msg, _ := link.Envelope(body, []byte("sig"))
named, err := namedByLatest(map[string][]byte{"novox": msg, "ace": []byte(`{"declaration":"` + "e30=" + `","signature":""}`)})
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
if err != nil || len(named) != 1 || !named[name] {
t.Fatalf("%v %v", named, err)
}
if _, err := namedByLatest(map[string][]byte{"novox": []byte("not json")}); err == nil {
t.Fatal("an unreadable latest declaration was read as naming none")
}
}
// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
const limit = 1048576
shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
n := wireSizeOf(t, shape)
t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
if n > limit/2 {
t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
}
if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
}
grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
if grown <= limit/2 {
t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
}
}
func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
+2 -2
View File
@@ -13,8 +13,8 @@ import (
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
// So the size is measured at every send: a warning condition from 90% of the limit, urgent over it, cleared when
// the declaration is back under 90%. The limit is the connection's own, not a constant.
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
// the declaration is back under 75%. The limit is the connection's own, not a constant.
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
const kindDeclarationNearLimit = "declaration-near-limit"
+8 -8
View File
@@ -33,7 +33,7 @@ func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
}
}
// From 90% of the bus's largest message the declaration is a warning; over it, urgent; back under 90%, cleared.
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
@@ -50,22 +50,22 @@ func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *test
return c, found
}
settleDeclarationRoom(t.Context(), k, "novox", 899, limit)
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
if _, found := read(); found {
t.Fatal("a declaration at 89.9% raised a condition")
t.Fatal("a declaration at 74.9% raised a condition")
}
settleDeclarationRoom(t.Context(), k, "novox", 900, limit)
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
c, found := read()
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "900 bytes") ||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
t.Fatalf("at 90%% the condition is %v %+v", found, c)
t.Fatalf("at 75%% the condition is %v %+v", found, c)
}
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
if c, _ := read(); c.Severity != conditions.Urgent {
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
}
settleDeclarationRoom(t.Context(), k, "novox", 850, limit)
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
if _, found := read(); found {
t.Fatal("the condition stayed after the declaration fell under 90%")
t.Fatal("the condition stayed after the declaration fell under 75%")
}
}
+238
View File
@@ -0,0 +1,238 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
// found named again, that is erased; found unnamed for a day, it is removed. Objects are immutable and regenerable from the
// controller's state, so a removal loses nothing.
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
const declarationUnnamed = "unnamed-since"
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
const sweepEvery = time.Hour
// bucketObject is one object of the bucket as the sweep sees it.
type bucketObject struct {
Name string
// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
UnnamedSince time.Time
}
// sweptBucket is what the sweep needs of the bucket.
type sweptBucket interface {
Objects(ctx context.Context) ([]bucketObject, error)
MarkUnnamed(ctx context.Context, name string, since time.Time) error
ClearUnnamed(ctx context.Context, name string) error
Delete(ctx context.Context, name string) error
}
// filesSweep says what a sweep did.
type filesSweep struct{ Kept, Marked, Cleared, Removed int }
// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
var r filesSweep
objects, err := bucket.Objects(ctx)
if err != nil {
return r, err
}
sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
for _, o := range objects {
switch {
case named[o.Name]:
r.Kept++
if !o.UnnamedSince.IsZero() {
if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Cleared++
}
case o.UnnamedSince.IsZero():
if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Marked++
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
if err := bucket.Delete(ctx, o.Name); err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Removed++
default:
r.Kept++
}
}
return r, nil
}
// namedByLatest is the name of every object any of these latest declarations names. A body that cannot be read is an error, never
// "names none": an object a node still needs would be marked for removal.
func namedByLatest(bodies map[string][]byte) (map[string]bool, error) {
named := map[string]bool{}
for node, signed := range bodies {
var s link.Signed
if err := json.Unmarshal(signed, &s); err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
names, err := namedObjects(s.Declaration)
if err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
for name := range names {
named[name] = true
}
}
return named, nil
}
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
// no entry.
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
js, err := conn.JetStream(nats.Context(ctx))
if err != nil {
return nil, err
}
out := map[string][]byte{}
for _, node := range nodes {
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
if errors.Is(err, nats.ErrMsgNotFound) {
continue
}
if err != nil {
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
}
out[node] = msg.Data
}
return out, nil
}
// busSweptBucket is the bucket on the controller's connection.
type busSweptBucket struct{ store jetstream.ObjectStore }
func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
infos, err := b.store.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return nil, nil
}
if err != nil {
return nil, err
}
out := make([]bucketObject, 0, len(infos))
for _, info := range infos {
o := bucketObject{Name: info.Name}
if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
o.UnnamedSince = at
}
out = append(out, o)
}
return out, nil
}
func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
info, err := b.store.GetInfo(ctx, name)
if err != nil {
return err
}
meta := info.ObjectMeta
kept := map[string]string{}
for k, v := range meta.Metadata {
kept[k] = v
}
change(kept)
meta.Metadata = kept
return b.store.UpdateMeta(ctx, name, meta)
}
func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
}
func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
}
func (b busSweptBucket) Delete(ctx context.Context, name string) error {
return b.store.Delete(ctx, name)
}
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
tick := time.NewTicker(sweepEvery)
defer tick.Stop()
failing := ""
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
continue
}
why := sweepOnce(ctx, inv, conn)
if why != failing {
if why != "" {
fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
} else {
fmt.Printf("the bucket of declaration files is swept again\n")
}
failing = why
}
}
}
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
nodes, err := inv.Nodes(working)
if err != nil {
return fmt.Sprintf("the nodes cannot be read: %v", err)
}
names := make([]string, 0, len(nodes))
for _, n := range nodes {
names = append(names, n.Name)
}
bodies, err := latestDeclarations(working, conn, names)
if err != nil {
return err.Error()
}
named, err := namedByLatest(bodies)
if err != nil {
return err.Error()
}
js, err := jetstream.New(conn)
if err != nil {
return err.Error()
}
store, err := js.ObjectStore(working, broker.DeclarationFilesBucket)
if err != nil {
return fmt.Sprintf("the bucket cannot be read: %v", err)
}
r, err := sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
if r.Removed+r.Marked+r.Cleared > 0 {
fmt.Printf("the bucket of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
r.Kept, r.Marked, r.Cleared, r.Removed)
}
if err != nil {
return err.Error()
}
return ""
}
+47
View File
@@ -282,6 +282,8 @@ func serve(ctx context.Context) (err error) {
// assignment.
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
go mirroringRecords(ctx, inv, bus.Conn, bus)
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
// to-be 45 §6).
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
@@ -979,6 +981,18 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
return err
}
digest, err := d.declare(ctx, s, body)
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
// references, and says so; the next one tries the bucket again.
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
s.declared.fileReferences = false
if body, err = s.declared.Body(); err != nil {
return err
}
digest, err = d.declare(ctx, s, body)
}
if err != nil {
return err
}
@@ -1040,6 +1054,34 @@ type overTheBus struct {
indent string
}
// putFiles puts the contents s names by digest in the bucket.
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
objects, err := s.declared.objects()
if err != nil {
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
}
if len(objects) == 0 {
return nil
}
bus, ok := b.server.Bus().(link.OverNATS)
if !ok {
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
}
store, err := openBucket(ctx, bus.Conn)
if err != nil {
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
}
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := putObjects(putting, store, objects); err != nil {
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
}
return nil
}
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
// when they cannot be, and never what holds the send back (assertOnSend).
@@ -1052,6 +1094,11 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
}
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
// that applies it finds every one. A node reading no references has none.
if err := b.putFiles(ctx, s); err != nil {
return "", err
}
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
return "", err
}
+27 -3
View File
@@ -39,6 +39,10 @@ type sendable struct {
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
ReadsModules bool
// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
// objects). Not on the wire as a field; the bodies differ.
fileReferences bool
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
// kept beside the resources, and on the wire only to a machine that reads them.
moduleOf map[string]string
@@ -115,12 +119,16 @@ type adoptionEnvelope struct {
// Body is the declaration's bytes, as sent and as digested.
func (s sendable) Body() ([]byte, error) {
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
resources, _, err := s.byReference()
if err != nil {
return nil, err
}
envelope := map[string]any{"declaration": 1, "resources": resources}
if s.ReadsModules {
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
named := make([]map[string]any, len(s.Resources))
for n, r := range s.Resources {
named := make([]map[string]any, len(resources))
for n, r := range resources {
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
if !owned {
named[n] = r
@@ -165,3 +173,19 @@ func (s sendable) Body() ([]byte, error) {
}
return json.Marshal(envelope)
}
// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
if !s.fileReferences {
return s.Resources, nil, nil
}
return referencing(s.Resources)
}
// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
// body is sent. Empty for a machine that is sent every content inline.
func (s sendable) objects() (map[string][]byte, error) {
_, objects, err := s.byReference()
return objects, err
}