From 3eef0254df9d0d2b46e1cd68a88701e847b8680a Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 01:59:38 +0100 Subject: [PATCH] H-A-227: answer an object store's fill from its stream's own figures The fill was a listing of every description in the store, deleted chunks' leftovers included; a receiver reading it per chunk made one connection read about 14 MB/s. The fill is now one stream info request: its bytes, its cap, and its live subjects as the count. Builds: design 47 H H-A-227 --- node-tools/internal/bus/bindings.go | 47 +++++++++++++------ node-tools/internal/bus/bindings_test.go | 5 ++ .../internal/bus/streams_internal_test.go | 17 +++++++ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/node-tools/internal/bus/bindings.go b/node-tools/internal/bus/bindings.go index 4a13023..7fb1d20 100644 --- a/node-tools/internal/bus/bindings.go +++ b/node-tools/internal/bus/bindings.go @@ -196,19 +196,28 @@ func (c *Conn) ObjectsList(module string, r buscalls.ObjectsRequest) (buscalls.O return out, nil } -// ObjectsInfo is how full the module's object store is: its objects, its bytes and its cap. +// ObjectsInfo is how full the module's object store is, from the figures the bus keeps for its stream: its bytes, its +// messages' live subjects as the count, and its cap. It never lists the store: a store's descriptions, the leftovers +// of deleted objects among them, may be many thousands, and a fill read per chunk of a transfer read them all each +// time (design 47 H lead answers H-A-227). func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) { - listed, err := c.ObjectsList(module, r) + b, err := c.objectBucketOf(module, r.Objects) if err != nil { return buscalls.Fill{}, err } - b, _ := c.objectBucketOf(module, r.Objects) - fill, err := c.streamFill("OBJ_" + b.ObjectStore) + info, err := c.streamInfo("OBJ_" + b.ObjectStore) if err != nil { return buscalls.Fill{}, err } - fill.Count = uint64(len(listed.Objects)) - return fill.Fill, nil + return objectsFillOf(info), nil +} + +// objectsFillOf is an object store's fill from its stream's info alone: the bytes the stream keeps, its live subjects +// as the count (H-A-227: the stream's live subjects, or none) and its cap. +func objectsFillOf(info *jetstream.StreamInfo) buscalls.Fill { + fill := streamFillOf(info).Fill + fill.Count = uint64(info.State.NumSubjects) + return fill } // ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted. @@ -261,29 +270,39 @@ func (c *Conn) ObjectsWatch(module string, r buscalls.ObjectsWatchRequest, deliv // streamFill is a stream's fill and its first and last sequence, by its name on the bus. func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) { + info, err := c.streamInfo(name) + if err != nil { + return buscalls.StreamFilled{}, err + } + return streamFillOf(info), nil +} + +// streamInfo is a stream's info as the bus keeps it, by its name on the bus: one request, whatever the stream holds. +func (c *Conn) streamInfo(name string) (*jetstream.StreamInfo, error) { ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() js, err := c.JetStream() if err != nil { - return buscalls.StreamFilled{}, err + return nil, err } s, err := js.Stream(ctx, name) if errors.Is(err, jetstream.ErrStreamNotFound) { - return buscalls.StreamFilled{}, fmt.Errorf("the stream %s is bound and not on the bus", name) + return nil, fmt.Errorf("the stream %s is bound and not on the bus", name) } if err != nil { - return buscalls.StreamFilled{}, err - } - info, err := s.Info(ctx) - if err != nil { - return buscalls.StreamFilled{}, err + return nil, err } + return s.Info(ctx) +} + +// streamFillOf is a stream's fill and its first and last sequence from its info. +func streamFillOf(info *jetstream.StreamInfo) buscalls.StreamFilled { max := uint64(0) if info.Config.MaxBytes > 0 { max = uint64(info.Config.MaxBytes) } return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max}, - First: info.State.FirstSeq, Last: info.State.LastSeq}, nil + First: info.State.FirstSeq, Last: info.State.LastSeq} } // streamBinding is the module's binding of a stream by its local name, a work queue among them. diff --git a/node-tools/internal/bus/bindings_test.go b/node-tools/internal/bus/bindings_test.go index 6521fb5..bc7c948 100644 --- a/node-tools/internal/bus/bindings_test.go +++ b/node-tools/internal/bus/bindings_test.go @@ -95,6 +95,11 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) { if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 { t.Fatalf("listed %+v, %v", listed, err) } + // H-A-227: the fill is the stream's own figures, its live subjects as the count (a's chunk and description, big's + // description left by its delete), never the count of its listed objects. + if fill, err := conn.ObjectsInfo("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || fill.Count != 3 || fill.Bytes == 0 { + t.Fatalf("objects fill %+v, %v", fill, err) + } // A-380: an old `objects` entry is reached by the store grants creates for it, `_`, with no binding of that name. if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil { t.Fatalf("put into the old objects section: %v", err) diff --git a/node-tools/internal/bus/streams_internal_test.go b/node-tools/internal/bus/streams_internal_test.go index 5b176c3..6ca0539 100644 --- a/node-tools/internal/bus/streams_internal_test.go +++ b/node-tools/internal/bus/streams_internal_test.go @@ -5,6 +5,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/buscalls" + "github.com/nats-io/nats.go/jetstream" ) // novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects. @@ -69,3 +70,19 @@ func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) { t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got) } } + +// Design 47 H lead answers H-A-227: an object store's fill is the figures the bus keeps for its stream, its live +// subjects as the count, never a count of its descriptions: a store holding twenty thousand messages, deleted chunks' +// leftovers among them, answers its bytes, its cap and its four live subjects. +func TestAnObjectStoresFillIsItsStreamsOwnFigures(t *testing.T) { + info := &jetstream.StreamInfo{ + Config: jetstream.StreamConfig{MaxBytes: 8 << 30}, + State: jetstream.StreamState{Msgs: 20653, Bytes: 1 << 30, NumSubjects: 4, FirstSeq: 9, LastSeq: 20661}, + } + if got, want := objectsFillOf(info), (buscalls.Fill{Count: 4, Bytes: 1 << 30, MaxBytes: 8 << 30}); got != want { + t.Fatalf("fill %+v, want %+v", got, want) + } + if got := objectsFillOf(&jetstream.StreamInfo{State: jetstream.StreamState{Bytes: 7}}); got != (buscalls.Fill{Bytes: 7}) { + t.Fatalf("a store with no cap and no live subject: fill %+v", got) + } +}