Send files over 64 KiB by digest through the bus's object bucket to nodes that read them, grant it, and warn at 75 percent (issues 1696, 1697, 1698; ADR 0486 step 3)

A node that reports file-references gets the digest and size of each file over 64 KiB that may travel so, and the content is put in the bucket declaration-files before the declaration is sent; others get every content inline. The controller alone writes the bucket and node-engines read an object by name (the four grants the node-engine's own fetch makes, no list, no write). An object goes a day after no node's latest declaration names it. The near-limit condition warns at 75 percent, and a test holds the control node's declaration, in the reference form, under half the bus's largest message.
2026-10-21 15:16:11 +02:00
parent fa5b5e1c11
commit ceaa616d01
18 changed files with 1486 additions and 18 deletions
@@ -0,0 +1,238 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
+// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
+// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
+// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
+// found named again, that is erased; found unnamed for a day, it is removed. Objects are immutable and regenerable from the
+// controller's state, so a removal loses nothing.
+
+// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
+const declarationUnnamed = "unnamed-since"
+
+// sweepEvery is how often the bucket is judged: the removal is exact to within this.
+const sweepEvery = time.Hour
+
+// bucketObject is one object of the bucket as the sweep sees it.
+type bucketObject struct {
+	Name string
+	// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
+	UnnamedSince time.Time
+}
+
+// sweptBucket is what the sweep needs of the bucket.
+type sweptBucket interface {
+	Objects(ctx context.Context) ([]bucketObject, error)
+	MarkUnnamed(ctx context.Context, name string, since time.Time) error
+	ClearUnnamed(ctx context.Context, name string) error
+	Delete(ctx context.Context, name string) error
+}
+
+// filesSweep says what a sweep did.
+type filesSweep struct{ Kept, Marked, Cleared, Removed int }
+
+// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
+// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
+func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
+	var r filesSweep
+	objects, err := bucket.Objects(ctx)
+	if err != nil {
+		return r, err
+	}
+	sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
+	for _, o := range objects {
+		switch {
+		case named[o.Name]:
+			r.Kept++
+			if !o.UnnamedSince.IsZero() {
+				if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
+					return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+				}
+				r.Cleared++
+			}
+		case o.UnnamedSince.IsZero():
+			if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
+				return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+			}
+			r.Marked++
+		case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
+			if err := bucket.Delete(ctx, o.Name); err != nil {
+				return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+			}
+			r.Removed++
+		default:
+			r.Kept++
+		}
+	}
+	return r, nil
+}
+
+// namedByLatest is the name of every object any of these latest declarations names. A body that cannot be read is an error, never
+// "names none": an object a node still needs would be marked for removal.
+func namedByLatest(bodies map[string][]byte) (map[string]bool, error) {
+	named := map[string]bool{}
+	for node, signed := range bodies {
+		var s link.Signed
+		if err := json.Unmarshal(signed, &s); err != nil {
+			return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+		}
+		names, err := namedObjects(s.Declaration)
+		if err != nil {
+			return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+		}
+		for name := range names {
+			named[name] = true
+		}
+	}
+	return named, nil
+}
+
+// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
+// no entry.
+func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
+	js, err := conn.JetStream(nats.Context(ctx))
+	if err != nil {
+		return nil, err
+	}
+	out := map[string][]byte{}
+	for _, node := range nodes {
+		msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
+		if errors.Is(err, nats.ErrMsgNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
+		}
+		out[node] = msg.Data
+	}
+	return out, nil
+}
+
+// busSweptBucket is the bucket on the controller's connection.
+type busSweptBucket struct{ store jetstream.ObjectStore }
+
+func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
+	infos, err := b.store.List(ctx)
+	if errors.Is(err, jetstream.ErrNoObjectsFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := make([]bucketObject, 0, len(infos))
+	for _, info := range infos {
+		o := bucketObject{Name: info.Name}
+		if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
+			o.UnnamedSince = at
+		}
+		out = append(out, o)
+	}
+	return out, nil
+}
+
+func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
+	info, err := b.store.GetInfo(ctx, name)
+	if err != nil {
+		return err
+	}
+	meta := info.ObjectMeta
+	kept := map[string]string{}
+	for k, v := range meta.Metadata {
+		kept[k] = v
+	}
+	change(kept)
+	meta.Metadata = kept
+	return b.store.UpdateMeta(ctx, name, meta)
+}
+
+func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
+	return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
+}
+
+func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
+	return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
+}
+
+func (b busSweptBucket) Delete(ctx context.Context, name string) error {
+	return b.store.Delete(ctx, name)
+}
+
+// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
+func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
+	tick := time.NewTicker(sweepEvery)
+	defer tick.Stop()
+	failing := ""
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+		if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
+			continue
+		}
+		why := sweepOnce(ctx, inv, conn)
+		if why != failing {
+			if why != "" {
+				fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
+			} else {
+				fmt.Printf("the bucket of declaration files is swept again\n")
+			}
+			failing = why
+		}
+	}
+}
+
+func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
+	working, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+	nodes, err := inv.Nodes(working)
+	if err != nil {
+		return fmt.Sprintf("the nodes cannot be read: %v", err)
+	}
+	names := make([]string, 0, len(nodes))
+	for _, n := range nodes {
+		names = append(names, n.Name)
+	}
+	bodies, err := latestDeclarations(working, conn, names)
+	if err != nil {
+		return err.Error()
+	}
+	named, err := namedByLatest(bodies)
+	if err != nil {
+		return err.Error()
+	}
+	js, err := jetstream.New(conn)
+	if err != nil {
+		return err.Error()
+	}
+	store, err := js.ObjectStore(working, broker.DeclarationFilesBucket)
+	if err != nil {
+		return fmt.Sprintf("the bucket cannot be read: %v", err)
+	}
+	r, err := sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
+	if r.Removed+r.Marked+r.Cleared > 0 {
+		fmt.Printf("the bucket of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
+			r.Kept, r.Marked, r.Cleared, r.Removed)
+	}
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}