novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Give each node a bucket of its own for the files that travel by digest, and keep the sweep from removing an object a send relies on (issues 1696, 1697; review of ADR 0486 step 3)

Browse Source 
A literal * in a published subject matches the grant's *, so in one shared bucket a node could read every node's files. Each node now
has declaration-files-<node>, which its account reads by name and the controller writes; the sweep judges each bucket against that
node's latest declaration, under the node's lock with the send, and an object the sweep marked is put again by the next send. Putting
that fails send after send raises a condition.
This commit is contained in:
jschoubben
2026-10-21 15:43:36 +02:00
parent ceaa616d01
commit da364c2730
15 changed files with 544 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/declaration_sweep.go
+76 -45
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
@@ -20,7 +21,8 @@ import (
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
// found named again, that is erased; found unnamed for a day, it is removed. Objects are immutable and regenerable from the
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
// node's latest declaration alone. Objects are immutable and regenerable from the
// controller's state, so a removal loses nothing.
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
@@ -41,7 +43,9 @@ type sweptBucket interface {
Objects(ctx context.Context) ([]bucketObject, error)
MarkUnnamed(ctx context.Context, name string, since time.Time) error
ClearUnnamed(ctx context.Context, name string) error
Delete(ctx context.Context, name string) error
// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
// since has erased the mark, and the object is left.
Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
}
// filesSweep says what a sweep did.
@@ -72,10 +76,15 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
}
r.Marked++
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
if err := bucket.Delete(ctx, o.Name); err != nil {
gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
if err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Removed++
if gone {
r.Removed++
} else {
r.Kept++
}
default:
r.Kept++
}
@@ -83,22 +92,20 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
return r, nil
}
// namedByLatest is the name of every object any of these latest declarations names. A body that cannot be read is an error, never
// "names none": an object a node still needs would be marked for removal.
func namedByLatest(bodies map[string][]byte) (map[string]bool, error) {
named := map[string]bool{}
for node, signed := range bodies {
var s link.Signed
if err := json.Unmarshal(signed, &s); err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
names, err := namedObjects(s.Declaration)
if err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
for name := range names {
named[name] = true
}
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
// "names none": an object the node still needs would be marked for removal.
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
var s link.Signed
if err := json.Unmarshal(signed, &s); err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
names, err := namedObjects(s.Declaration)
if err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
named := make(map[string]bool, len(names))
for name := range names {
named[name] = true
}
return named, nil
}
@@ -169,8 +176,18 @@ func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
}
func (b busSweptBucket) Delete(ctx context.Context, name string) error {
return b.store.Delete(ctx, name)
func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
info, err := b.store.GetInfo(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return false, nil
}
if err != nil {
return false, err
}
if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
return false, nil
}
return true, b.store.Delete(ctx, name)
}
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
@@ -199,40 +216,54 @@ func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, con
}
}
// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
// is waiting for the lock.
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
nodes, err := inv.Nodes(working)
nodes, err := inv.Nodes(ctx)
if err != nil {
return fmt.Sprintf("the nodes cannot be read: %v", err)
}
names := make([]string, 0, len(nodes))
var problems []string
var total filesSweep
for _, n := range nodes {
names = append(names, n.Name)
r, err := sweepNode(ctx, conn, n.Name)
total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
if err != nil {
problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
}
}
bodies, err := latestDeclarations(working, conn, names)
if err != nil {
return err.Error()
}
named, err := namedByLatest(bodies)
if err != nil {
return err.Error()
if total.Removed+total.Marked+total.Cleared > 0 {
fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
total.Kept, total.Marked, total.Cleared, total.Removed)
}
return strings.Join(problems, "; ")
}
func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
js, err := jetstream.New(conn)
if err != nil {
return err.Error()
return filesSweep{}, err
}
store, err := js.ObjectStore(working, broker.DeclarationFilesBucket)
if err != nil {
return fmt.Sprintf("the bucket cannot be read: %v", err)
}
r, err := sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
if r.Removed+r.Marked+r.Cleared > 0 {
fmt.Printf("the bucket of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
r.Kept, r.Marked, r.Cleared, r.Removed)
store, err := js.ObjectStore(working, broker.DeclarationFilesBucketOf(node))
if errors.Is(err, jetstream.ErrBucketNotFound) {
return filesSweep{}, nil // a node never sent a reference has no bucket yet
}
if err != nil {
return err.Error()
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
}
return ""
defer lockFilesOf(node)()
latest, err := latestDeclarations(working, conn, []string{node})
if err != nil {
return filesSweep{}, err
}
named := map[string]bool{}
if body, ok := latest[node]; ok {
if named, err = namedByLatest(node, body); err != nil {
return filesSweep{}, err
}
}
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
}