From d6c516ab445a84ead70dccb850baeb30da588c73 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:34:32 +0100 Subject: [PATCH 01/12] Serve a bundle its own bound objects, streams and work queues on its pipe (issue 1966 T1) --- node-tools/go.mod | 2 + node-tools/go.sum | 2 - node-tools/internal/bus/bindings.go | 398 ++++++++++++++++++ node-tools/internal/bus/bindings_test.go | 112 +++++ node-tools/internal/bus/bus.go | 3 + .../internal/launch/event_answer_test.go | 4 + node-tools/internal/launch/launch.go | 45 ++ node-tools/internal/runtime/busstate.go | 102 +++++ 8 files changed, 666 insertions(+), 2 deletions(-) create mode 100644 node-tools/internal/bus/bindings.go create mode 100644 node-tools/internal/bus/bindings_test.go create mode 100644 node-tools/internal/runtime/busstate.go diff --git a/node-tools/go.mod b/node-tools/go.mod index 4b2d0b9..f2cc3df 100644 --- a/node-tools/go.mod +++ b/node-tools/go.mod @@ -20,3 +20,5 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-sdk/go => ../../mesh-sdk/go diff --git a/node-tools/go.sum b/node-tools/go.sum index 43cc4ec..98dc72c 100644 --- a/node-tools/go.sum +++ b/node-tools/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/node-tools/internal/bus/bindings.go b/node-tools/internal/bus/bindings.go new file mode 100644 index 0000000..8937a61 --- /dev/null +++ b/node-tools/internal/bus/bindings.go @@ -0,0 +1,398 @@ +package bus + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats.go/jetstream" +) + +// A module's bus state in the new form (novox/hq ADR 0595, design 25 § 12.5; issue 1966 T1): an object store, a stream, +// a work queue, a ledger, each asked for in `requires` by a local name and bound in the module's membership by the bus +// names the bus module created. **A bundle reaches only its own module's bindings**: one account per machine carries +// every module on it, so what a module may reach is decided here, from its membership, and refused with the reason +// before anything is sent, as its state is (state.go). + +// Binding is the module's binding of a local name, of the storage type storage, or the reason there is none. +func (c *Conn) Binding(module, local, storage string) (membership.BusBinding, error) { + m := c.Membership(module) + if m == nil { + return membership.BusBinding{}, fmt.Errorf("%s has no membership issued on %s yet, so none of its bus state is "+ + "reachable until the mesh issues one (novox/hq ADR 0595)", module, c.node) + } + var names []string + for _, b := range m.Bindings { + if b.Local != local { + names = append(names, b.Local+" ("+b.Storage+")") + continue + } + if b.Storage != storage { + return membership.BusBinding{}, fmt.Errorf("%s's %q is bound with the storage type %s, not %s (novox/hq ADR 0595)", + module, local, b.Storage, storage) + } + return b, nil + } + sort.Strings(names) + bound := "none" + if len(names) > 0 { + bound = strings.Join(names, ", ") + } + return membership.BusBinding{}, fmt.Errorf("%s was bound no %s called %q: a module reaches only the bus state its "+ + "manifest asks for in `requires`, by its local name, once the mesh created it; it was bound: %s (novox/hq ADR 0595)", + module, storage, local, bound) +} + +// Bindings is every binding a module's membership carries now. +func (c *Conn) Bindings(module string) []membership.BusBinding { + if m := c.Membership(module); m != nil { + return append([]membership.BusBinding(nil), m.Bindings...) + } + return nil +} + +// jetStream is a JetStream handle on this connection. +func (c *Conn) jetStream() (jetstream.JetStream, error) { return jetstream.New(c.nc) } + +func (c *Conn) objectStore(ctx context.Context, module, local string) (jetstream.ObjectStore, membership.BusBinding, error) { + b, err := c.Binding(module, local, catalogue.StorageObjectStore) + if err != nil { + return nil, b, err + } + js, err := c.jetStream() + if err != nil { + return nil, b, err + } + obs, err := js.ObjectStore(ctx, b.ObjectStore) + if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, b, fmt.Errorf("%s's objects %q are bound and their store %s is not on the bus", module, local, b.ObjectStore) + } + return obs, b, err +} + +func objectInfoOf(i *jetstream.ObjectInfo) buscalls.ObjectInfo { + return buscalls.ObjectInfo{Name: i.Name, Size: i.Size, Chunks: i.Chunks, Digest: i.Digest, Description: i.Description, + Modified: i.ModTime, Deleted: i.Deleted} +} + +// ObjectsPut stores one object whole under its name in the module's object store. +func (c *Conn) ObjectsPut(module string, r buscalls.ObjectsPutRequest) (buscalls.ObjectInfo, error) { + if r.Name == "" { + return buscalls.ObjectInfo{}, errors.New("an object is put under a name") + } + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + obs, _, err := c.objectStore(ctx, module, r.Objects) + if err != nil { + return buscalls.ObjectInfo{}, err + } + info, err := obs.PutBytes(ctx, r.Name, r.Data) + if err != nil { + return buscalls.ObjectInfo{}, err + } + if r.Description != "" { + if err := obs.UpdateMeta(ctx, r.Name, jetstream.ObjectMeta{Name: r.Name, Description: r.Description}); err != nil { + return buscalls.ObjectInfo{}, err + } + if info, err = obs.GetInfo(ctx, r.Name); err != nil { + return buscalls.ObjectInfo{}, err + } + } + return objectInfoOf(info), nil +} + +// ObjectsGet is one object of the module's object store, its info and its bytes; nil when there is none. +func (c *Conn) ObjectsGet(module string, r buscalls.ObjectsNameRequest) (*buscalls.ObjectData, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + obs, _, err := c.objectStore(ctx, module, r.Objects) + if err != nil { + return nil, err + } + res, err := obs.Get(ctx, r.Name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + defer res.Close() + info, err := res.Info() + if err != nil { + return nil, err + } + data, err := obs.GetBytes(ctx, r.Name) + if err != nil { + return nil, err + } + return &buscalls.ObjectData{Info: objectInfoOf(info), Data: data}, nil +} + +// ObjectsDelete deletes one object of the module's object store; one that is not there is not an error. +func (c *Conn) ObjectsDelete(module string, r buscalls.ObjectsNameRequest) error { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + obs, _, err := c.objectStore(ctx, module, r.Objects) + if err != nil { + return err + } + if err := obs.Delete(ctx, r.Name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +// ObjectsList is every object the module's object store holds. +func (c *Conn) ObjectsList(module string, r buscalls.ObjectsRequest) (buscalls.ObjectsListed, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + obs, _, err := c.objectStore(ctx, module, r.Objects) + if err != nil { + return buscalls.ObjectsListed{}, err + } + out := buscalls.ObjectsListed{Objects: []buscalls.ObjectInfo{}} + infos, err := obs.List(ctx) + if errors.Is(err, jetstream.ErrNoObjectsFound) { + return out, nil + } + if err != nil { + return out, err + } + for _, i := range infos { + out.Objects = append(out.Objects, objectInfoOf(i)) + } + return out, nil +} + +// ObjectsInfo is how full the module's object store is: its objects, its bytes and its cap. +func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) { + listed, err := c.ObjectsList(module, r) + if err != nil { + return buscalls.Fill{}, err + } + b, _ := c.Binding(module, r.Objects, catalogue.StorageObjectStore) + fill, err := c.streamFill("OBJ_" + b.ObjectStore) + if err != nil { + return buscalls.Fill{}, err + } + fill.Count = uint64(len(listed.Objects)) + return fill.Fill, nil +} + +// ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted. +func (c *Conn) ObjectsWatch(module string, r buscalls.ObjectsWatchRequest, deliver func(buscalls.ObjectsChange) error) (func(), error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + obs, _, err := c.objectStore(ctx, module, r.Objects) + cancel() + if err != nil { + return nil, err + } + watchCtx, stop := context.WithCancel(context.Background()) + w, err := obs.Watch(watchCtx, jetstream.IncludeHistory()) + if err != nil { + stop() + return nil, err + } + current := true + for current { + select { + case i, ok := <-w.Updates(): + if !ok { + stop() + return nil, errors.New("the object store's watch ended before its current objects were handed over") + } + if i == nil { + current = false + continue + } + if err := deliver(buscalls.ObjectsChange{Watch: r.Watch, Object: objectInfoOf(i), Current: true}); err != nil { + c.Logf("[mesh-tools] %s did not take an object of %s: %v", module, r.Objects, err) + } + case <-time.After(StateTimeout): + _ = w.Stop() + stop() + return nil, fmt.Errorf("the object store %s did not hand its current objects in %s", r.Objects, StateTimeout) + } + } + go func() { + for i := range w.Updates() { + if i == nil { + continue + } + if err := deliver(buscalls.ObjectsChange{Watch: r.Watch, Object: objectInfoOf(i)}); err != nil { + c.Logf("[mesh-tools] %s did not take an object of %s: %v", module, r.Objects, err) + } + } + }() + return func() { _ = w.Stop(); stop() }, nil +} + +// streamFill is a stream's fill and its first and last sequence, by its name on the bus. +func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + js, err := c.jetStream() + if err != nil { + return buscalls.StreamFilled{}, err + } + s, err := js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return buscalls.StreamFilled{}, fmt.Errorf("the stream %s is bound and not on the bus", name) + } + if err != nil { + return buscalls.StreamFilled{}, err + } + info, err := s.Info(ctx) + if err != nil { + return buscalls.StreamFilled{}, err + } + max := uint64(0) + if info.Config.MaxBytes > 0 { + max = uint64(info.Config.MaxBytes) + } + return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max}, + First: info.State.FirstSeq, Last: info.State.LastSeq}, nil +} + +// streamBinding is the module's binding of a stream by its local name, a work queue among them. +func (c *Conn) streamBinding(module, local string) (membership.BusBinding, error) { + return c.Binding(module, local, catalogue.StorageStream) +} + +// StreamFill is how full the module's stream is. +func (c *Conn) StreamFill(module string, r buscalls.StreamRequest) (buscalls.StreamFilled, error) { + b, err := c.streamBinding(module, r.Stream) + if err != nil { + return buscalls.StreamFilled{}, err + } + return c.streamFill(b.Stream) +} + +// StreamReadMost is the most messages one read answers. +const StreamReadMost = 1000 + +// StreamRead is the module's stream's messages from a sequence on, Subjects narrowing them, at most Limit of them. +func (c *Conn) StreamRead(module string, r buscalls.StreamReadRequest) (buscalls.StreamPage, error) { + b, err := c.streamBinding(module, r.Stream) + if err != nil { + return buscalls.StreamPage{}, err + } + return c.readStream(b.Stream, r.Subjects, r.From, r.Limit) +} + +// readStream reads a stream by its bus name from a sequence on, the messages on the subjects filter names (every +// subject when empty), at most limit of them. +func (c *Conn) readStream(name, filter string, from uint64, limit int) (buscalls.StreamPage, error) { + if limit <= 0 || limit > StreamReadMost { + limit = StreamReadMost + } + if filter == "" { + filter = ">" + } + if from == 0 { + from = 1 + } + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + js, err := c.jetStream() + if err != nil { + return buscalls.StreamPage{}, err + } + s, err := js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return buscalls.StreamPage{}, fmt.Errorf("the stream %s is bound and not on the bus", name) + } + if err != nil { + return buscalls.StreamPage{}, err + } + info, err := s.Info(ctx) + if err != nil { + return buscalls.StreamPage{}, err + } + page := buscalls.StreamPage{Messages: []buscalls.StreamMessage{}, Last: info.State.LastSeq} + for seq := from; seq <= info.State.LastSeq && len(page.Messages) < limit; { + msg, err := s.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(filter)) + if errors.Is(err, jetstream.ErrMsgNotFound) { + break + } + if err != nil { + return page, err + } + body := json.RawMessage(msg.Data) + if !json.Valid(msg.Data) { + body, _ = json.Marshal(string(msg.Data)) + } + page.Messages = append(page.Messages, buscalls.StreamMessage{Sequence: msg.Sequence, Subject: msg.Subject, + Body: body, At: msg.Time}) + seq = msg.Sequence + 1 + if len(page.Messages) == limit && seq <= info.State.LastSeq { + page.Next = seq + } + } + return page, nil +} + +// WorkQueueSize is how many messages wait in the module's work queue, against its cap. +func (c *Conn) WorkQueueSize(module string, r buscalls.WorkQueueRequest) (buscalls.WorkQueueSized, error) { + b, err := c.streamBinding(module, r.Queue) + if err != nil { + return buscalls.WorkQueueSized{}, err + } + fill, err := c.streamFill(b.Stream) + if err != nil { + return buscalls.WorkQueueSized{}, err + } + return buscalls.WorkQueueSized{Fill: fill.Fill}, nil +} + +// WorkQueueDead is the module's work queue's dead letters: in a work queue a message is gone once settled, so one still +// kept below a consumer's acknowledgement floor was delivered as often as the consumer allows and never settled. +func (c *Conn) WorkQueueDead(module string, r buscalls.WorkQueueDeadRequest) (buscalls.WorkQueueDeadLetters, error) { + b, err := c.streamBinding(module, r.Queue) + if err != nil { + return buscalls.WorkQueueDeadLetters{}, err + } + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + js, err := c.jetStream() + if err != nil { + return buscalls.WorkQueueDeadLetters{}, err + } + s, err := js.Stream(ctx, b.Stream) + if err != nil { + return buscalls.WorkQueueDeadLetters{}, fmt.Errorf("the work queue %s: %w", b.Stream, err) + } + out := buscalls.WorkQueueDeadLetters{Dead: []buscalls.DeadLetter{}} + consumers := s.ListConsumers(ctx) + for ci := range consumers.Info() { + floor := ci.AckFloor.Stream + if ci.Config.MaxDeliver <= 0 || floor == 0 { + continue + } + page, err := c.readStream(b.Stream, ci.Config.FilterSubject, 1, r.Limit) + if err != nil { + return out, err + } + for _, m := range page.Messages { + if m.Sequence > floor { + break + } + out.Dead = append(out.Dead, buscalls.DeadLetter{Sequence: m.Sequence, Subject: m.Subject, + Deliveries: ci.Config.MaxDeliver, Body: m.Body, At: m.At}) + } + } + if err := consumers.Err(); err != nil { + return out, err + } + if r.Limit > 0 && len(out.Dead) > r.Limit { + out.Dead = out.Dead[:r.Limit] + } + return out, nil +} diff --git a/node-tools/internal/bus/bindings_test.go b/node-tools/internal/bus/bindings_test.go new file mode 100644 index 0000000..81c25a0 --- /dev/null +++ b/node-tools/internal/bus/bindings_test.go @@ -0,0 +1,112 @@ +package bus_test + +import ( + "context" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-tools/node-tools/internal/bus" + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// 1966/AC12: given a bundle whose membership binds objects `files`, when it puts and gets `a`, then it reads back what it +// put; a local name it was not bound is refused with the reason, and so is another module's binding and a binding of +// another storage type. A stream and a work queue bound alike answer their read-back, fill and size. +func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) { + mesh := mt.New(t) + nc, err := nats.Connect(mt.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, _ := jetstream.New(nc) + ctx := context.Background() + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "m_files"}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "m_events", Subjects: []string{"mesh.state.m.events.>"}}); err != nil { + t.Fatal(err) + } + for _, s := range []string{"mesh.state.m.events.one", "mesh.state.m.events.two"} { + if _, err := js.Publish(ctx, s, []byte(`{"n":1}`)); err != nil { + t.Fatal(err) + } + } + m := mt.MembershipOf("m", "anchor", false, nil) + m.Bindings = []membership.BusBinding{ + {Local: "files", Storage: catalogue.StorageObjectStore, Template: catalogue.TemplateObjects, ObjectStore: "m_files"}, + {Local: "events", Storage: catalogue.StorageStream, Template: catalogue.TemplateEvents, Stream: "m_events", + Subject: "mesh.state.m.events"}, + } + other := mt.MembershipOf("other", "anchor", false, nil) + mesh.Issue(t, m) + mesh.Issue(t, other) + conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"}) + if err != nil { + t.Fatal(err) + } + conn.Logf = func(string, ...any) {} + t.Cleanup(conn.Close) + conn.Follow("m") + conn.Follow("other") + mt.Until(t, func() error { + if conn.Membership("m") == nil || conn.Membership("other") == nil { + return errors.New("not issued yet") + } + return nil + }) + + if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "files", Name: "a", Data: []byte("what was put")}); err != nil { + t.Fatal(err) + } + got, err := conn.ObjectsGet("m", buscalls.ObjectsNameRequest{Objects: "files", Name: "a"}) + if err != nil || got == nil || string(got.Data) != "what was put" || got.Info.Name != "a" { + t.Fatalf("read back %+v, %v", got, err) + } + if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 { + t.Fatalf("listed %+v, %v", listed, err) + } + page, err := conn.StreamRead("m", buscalls.StreamReadRequest{Stream: "events", Limit: 1}) + if err != nil || len(page.Messages) != 1 || page.Next != 2 || page.Last != 2 { + t.Fatalf("read %+v, %v", page, err) + } + if fill, err := conn.StreamFill("m", buscalls.StreamRequest{Stream: "events"}); err != nil || fill.Count != 2 { + t.Fatalf("fill %+v, %v", fill, err) + } + + for _, c := range []struct { + name, module, refusal string + call func(module string) error + }{ + {"a local name not bound", "m", `bound no object-store called "photos"`, func(module string) error { + _, err := conn.ObjectsGet(module, buscalls.ObjectsNameRequest{Objects: "photos", Name: "a"}) + return err + }}, + {"another module's binding", "other", `other was bound no object-store called "files"`, func(module string) error { + _, err := conn.ObjectsGet(module, buscalls.ObjectsNameRequest{Objects: "files", Name: "a"}) + return err + }}, + {"a binding of another storage type", "m", `"events" is bound with the storage type stream, not object-store`, func(module string) error { + _, err := conn.ObjectsList(module, buscalls.ObjectsRequest{Objects: "events"}) + return err + }}, + {"a work queue not bound", "m", `bound no stream called "jobs"`, func(module string) error { + _, err := conn.WorkQueueSize(module, buscalls.WorkQueueRequest{Queue: "jobs"}) + return err + }}, + } { + t.Run(c.name, func(t *testing.T) { + err := c.call(c.module) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Fatalf("refused with %v, want the reason %q", err, c.refusal) + } + }) + } +} diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index fa1a7c3..8027e53 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -78,6 +78,9 @@ type Membership struct { // Waits are the event families a session may wait on through the mesh MCP server (novox/hq ADR 0367 §2): // the tool runner's alone, composed from its manifest with the grant that lets it subscribe them. Waits []WaitIssued `json:"waits,omitempty"` + // Bindings are the module's bus state in the new form, by local name (novox/hq ADR 0595 §1, mesh-sdk + // membership.BusBinding): its object stores, streams, work queues and ledgers, its own alone (bindings.go). + Bindings []membership.BusBinding `json:"bindings,omitempty"` } // Served is an address a tool is answered on; `{tool}` stands for the tool's name. diff --git a/node-tools/internal/launch/event_answer_test.go b/node-tools/internal/launch/event_answer_test.go index 7ccfc47..6746b36 100644 --- a/node-tools/internal/launch/event_answer_test.go +++ b/node-tools/internal/launch/event_answer_test.go @@ -48,6 +48,10 @@ func (b *subscribing) Provision(string, json.RawMessage) (json.RawMessage, error func (b *subscribing) ProvisionWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) { return func() {}, nil } +func (b *subscribing) BusState(string, json.RawMessage) (json.RawMessage, error) { return nil, nil } +func (b *subscribing) BusStateWatch(string, json.RawMessage, func(json.RawMessage) error) (func(), error) { + return func() {}, nil +} func (b *subscribing) Subscribe(d func(json.RawMessage) error) error { b.mu.Lock() defer b.mu.Unlock() diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 43f3fff..0b91f71 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/buscalls" "golang.org/x/sys/unix" "github.com/novox/mesh-tools/node-tools/internal/wire" @@ -91,8 +92,26 @@ type Bus interface { // (novox/hq ADR 0349 §14, ADR 0370 §4): its config, and its bindings' values that are not secret. Provision(verb string, params json.RawMessage) (json.RawMessage, error) ProvisionWatch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error) + // BusState answers a bundle's calls on its module's bus state in the new form (mesh-sdk buscalls; novox/hq ADR 0595 + // §9, design 25 § 12.5): its objects, a stream's read-back and fill, a work queue's size and dead letters, and its + // ledgers, by the method the bundle asked, from its own module's bindings alone. + BusState(method string, params json.RawMessage) (json.RawMessage, error) + // BusStateWatch hands deliver each change of an objects or a ledger watch, the current ones first. + BusStateWatch(method string, params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error) } +// busStateCalls are the methods of mesh-sdk buscalls a bundle asks and BusState answers. +var busStateCalls = map[string]bool{ + buscalls.ObjectsPut: true, buscalls.ObjectsGet: true, buscalls.ObjectsList: true, buscalls.ObjectsDelete: true, + buscalls.ObjectsInfo: true, buscalls.StreamRead: true, buscalls.StreamFill: true, buscalls.WorkQueueSize: true, + buscalls.WorkQueueDead: true, buscalls.LedgerApply: true, buscalls.LedgerGet: true, buscalls.LedgerHistory: true, + buscalls.LedgerQuery: true, buscalls.LedgerRebuild: true, buscalls.LedgerReindex: true, buscalls.LedgerSnapshot: true, + buscalls.LedgerLag: true, +} + +// busStateWatches are the watches of mesh-sdk buscalls, each with the method its changes are handed to the bundle by. +var busStateWatches = map[string]string{buscalls.ObjectsWatch: buscalls.ObjectsWatched, buscalls.LedgerWatch: buscalls.LedgerWatched} + // EventTimeout bounds how long a bundle has to handle one event before it is offered again. var EventTimeout = 2 * time.Minute @@ -399,7 +418,33 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an c.mu.Unlock() } } + case buscalls.ObjectsWatch, buscalls.LedgerWatch: + // As a state's watch: each change asked of this child in order, by the method of its + // family; the change names the watch by the id the bundle gave in its request. + var stop func() + told := busStateWatches[m.Method] + stop, err = mesh.BusStateWatch(m.Method, m.Params, func(change json.RawMessage) error { + _, err := c.ask(module, told, change, EventTimeout) + return err + }) + if err == nil { + c.mu.Lock() + if c.why != nil { + c.mu.Unlock() + stop() + } else { + c.watches = append(c.watches, stop) + c.mu.Unlock() + } + } default: + if busStateCalls[m.Method] { + var answered json.RawMessage + if answered, err = mesh.BusState(m.Method, m.Params); err == nil { + result = answered + } + break + } if len(m.ID) > 0 { _ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID, "error": map[string]any{"code": -32601, "message": "the runtime answers no " + m.Method + " from a bundle"}}) diff --git a/node-tools/internal/runtime/busstate.go b/node-tools/internal/runtime/busstate.go new file mode 100644 index 0000000..905d3f0 --- /dev/null +++ b/node-tools/internal/runtime/busstate.go @@ -0,0 +1,102 @@ +package runtime + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// BusState answers a bundle's calls on its module's bus state in the new form (mesh-sdk buscalls; novox/hq ADR 0595 +// §9, design 25 § 12.5; issue 1966): each as the module whose bundle asks, never a module the request names, so a +// bundle reaches its own module's bindings alone. +func (b *moduleBus) BusState(method string, params json.RawMessage) (json.RawMessage, error) { + conn := b.all.conn + var answer any + var err error + switch method { + case buscalls.ObjectsPut: + var r buscalls.ObjectsPutRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.ObjectsPut(b.module, r) + } + case buscalls.ObjectsGet: + var r buscalls.ObjectsNameRequest + if err = decode(method, params, &r); err == nil { + var got *buscalls.ObjectData + if got, err = conn.ObjectsGet(b.module, r); err == nil && got == nil { + return json.RawMessage("null"), nil + } + answer = got + } + case buscalls.ObjectsDelete: + var r buscalls.ObjectsNameRequest + if err = decode(method, params, &r); err == nil { + err = conn.ObjectsDelete(b.module, r) + answer = buscalls.Done{} + } + case buscalls.ObjectsList: + var r buscalls.ObjectsRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.ObjectsList(b.module, r) + } + case buscalls.ObjectsInfo: + var r buscalls.ObjectsRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.ObjectsInfo(b.module, r) + } + case buscalls.StreamRead: + var r buscalls.StreamReadRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.StreamRead(b.module, r) + } + case buscalls.StreamFill: + var r buscalls.StreamRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.StreamFill(b.module, r) + } + case buscalls.WorkQueueSize: + var r buscalls.WorkQueueRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.WorkQueueSize(b.module, r) + } + case buscalls.WorkQueueDead: + var r buscalls.WorkQueueDeadRequest + if err = decode(method, params, &r); err == nil { + answer, err = conn.WorkQueueDead(b.module, r) + } + default: + return nil, fmt.Errorf("the runtime answers no %s", method) + } + if err != nil { + return nil, err + } + return json.Marshal(answer) +} + +// BusStateWatch hands a bundle each change of its objects or its ledger, the current ones first. +func (b *moduleBus) BusStateWatch(method string, params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) { + switch method { + case buscalls.ObjectsWatch: + var r buscalls.ObjectsWatchRequest + if err := decode(method, params, &r); err != nil { + return nil, err + } + return b.all.conn.ObjectsWatch(b.module, r, func(change buscalls.ObjectsChange) error { + raw, err := json.Marshal(change) + if err != nil { + return err + } + return deliver(raw) + }) + } + return nil, fmt.Errorf("the runtime answers no %s", method) +} + +// decode reads a bus-state call's request. +func decode(method string, params json.RawMessage, into any) error { + if err := json.Unmarshal(params, into); err != nil { + return fmt.Errorf("%s's request is not of its shape (mesh-sdk buscalls): %w", method, err) + } + return nil +} -- 2.54.0 From ad22c5e9fe7c2162d50bda4b16bc0a42682814b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:40:40 +0100 Subject: [PATCH 02/12] Apply a ledger operation: check it against its type, append it with the last sequence expected, derive the state at the revision read, write the snapshot (issue 1966 T2) --- node-tools/internal/bus/ledger.go | 229 ++++++++++ node-tools/internal/ledger/ledger.go | 493 ++++++++++++++++++++++ node-tools/internal/ledger/ledger_test.go | 102 +++++ node-tools/internal/ledger/memory.go | 158 +++++++ node-tools/internal/ledger/types_test.go | 60 +++ 5 files changed, 1042 insertions(+) create mode 100644 node-tools/internal/bus/ledger.go create mode 100644 node-tools/internal/ledger/ledger.go create mode 100644 node-tools/internal/ledger/ledger_test.go create mode 100644 node-tools/internal/ledger/memory.go create mode 100644 node-tools/internal/ledger/types_test.go diff --git a/node-tools/internal/bus/ledger.go b/node-tools/internal/bus/ledger.go new file mode 100644 index 0000000..bf46d53 --- /dev/null +++ b/node-tools/internal/bus/ledger.go @@ -0,0 +1,229 @@ +package bus + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-tools/node-tools/internal/ledger" +) + +// A ledger on the bus (novox/hq ADR 0597, design 25 § 12.4; issue 1966): its history a stream with a subject per +// instance under the binding's subject, its state a key-value bucket, its snapshots a subject per instance under the +// binding's snapshot subject, the latest only (the stream that holds them keeps one per subject, as the bus module +// creates it). **Only the module's ledger connection publishes on its history and its snapshots** (ADR 0597 §9): what +// is read, and the state, go on the runner's own. + +// LedgerStores are the stores of one ledger binding: reads and the state on c, the history's appends and the snapshots' +// writes on writer, the module's ledger connection. +func (c *Conn) LedgerStores(b membership.BusBinding, writer *Conn) (ledger.History, ledger.State, ledger.Snapshots) { + return &busHistory{read: c, write: writer, stream: b.History, prefix: prefixOf(b.Subject)}, + &busState{c: c, bucket: b.State}, + &busSnapshots{read: c, write: writer, prefix: prefixOf(b.Snapshot)} +} + +// prefixOf is a binding's subject as the prefix an instance's key is added to: written with or without its `.>`. +func prefixOf(subject string) string { + return strings.TrimSuffix(strings.TrimSuffix(subject, ">"), ".") +} + +// publishExpecting publishes body on subject on the connection, written only when the subject's last message is still +// at expect (0: it has none); ledger.ErrRaced when it is not. +func publishExpecting(c *Conn, subject string, body []byte, expect uint64) (uint64, error) { + msg := nats.NewMsg(subject) + msg.Data = body + msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(expect, 10)) + reply, err := c.nc.RequestMsg(msg, StateTimeout) + if errors.Is(err, nats.ErrNoResponders) { + return 0, fmt.Errorf("no stream on the bus keeps %s", subject) + } + if err != nil { + return 0, err + } + var ack struct { + Seq uint64 `json:"seq"` + Error *struct { + ErrCode uint16 `json:"err_code"` + Description string `json:"description"` + } `json:"error"` + } + if err := json.Unmarshal(reply.Data, &ack); err != nil { + return 0, fmt.Errorf("the bus answered the write to %s with what is not an acknowledgement: %q", subject, reply.Data) + } + if ack.Error != nil { + if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence || + code == jetstream.JSErrCodeStreamWrongLastSequenceConstant { + return 0, ledger.ErrRaced + } + return 0, fmt.Errorf("the bus refused the write to %s: %s (%d)", subject, ack.Error.Description, ack.Error.ErrCode) + } + return ack.Seq, nil +} + +// lastOn is the last message on a subject of the stream that keeps it: its bytes and sequence, nil and 0 for none. +func lastOn(c *Conn, stream, subject string) ([]byte, uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + js, err := c.jetStream() + if err != nil { + return nil, 0, err + } + if stream == "" { + if stream, err = js.StreamNameBySubject(ctx, subject); err != nil { + return nil, 0, fmt.Errorf("no stream on the bus keeps %s: %w", subject, err) + } + } + s, err := js.Stream(ctx, stream) + if err != nil { + return nil, 0, fmt.Errorf("the stream %s: %w", stream, err) + } + m, err := s.GetLastMsgForSubject(ctx, subject) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + return m.Data, m.Sequence, nil +} + +type busHistory struct { + read, write *Conn + stream, prefix string +} + +func (h *busHistory) Last(key string) (uint64, error) { + _, seq, err := lastOn(h.read, h.stream, h.prefix+"."+key) + return seq, err +} + +func (h *busHistory) Append(key string, op []byte, expect uint64) (uint64, error) { + if h.write == nil { + return 0, errors.New("the ledger's history is published only on its module's ledger connection, and none is held") + } + return publishExpecting(h.write, h.prefix+"."+key, op, expect) +} + +func (h *busHistory) Read(keys string, from uint64, limit int) ([]ledger.Kept, error) { + var out []ledger.Kept + for { + want := StreamReadMost + if limit > 0 && limit-len(out) < want { + want = limit - len(out) + } + page, err := h.read.readStream(h.stream, h.prefix+"."+keys, from, want) + if err != nil { + return out, err + } + for _, m := range page.Messages { + out = append(out, ledger.Kept{Sequence: m.Sequence, Data: m.Body}) + } + if page.Next == 0 || (limit > 0 && len(out) >= limit) { + return out, nil + } + from = page.Next + } +} + +type busState struct { + c *Conn + bucket string +} + +func (s *busState) kv(ctx context.Context) (jetstream.KeyValue, error) { + js, err := s.c.jetStream() + if err != nil { + return nil, err + } + kv, err := js.KeyValue(ctx, s.bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("the ledger's state %s is not on the bus", s.bucket) + } + return kv, err +} + +func (s *busState) Get(key string) ([]byte, uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + kv, err := s.kv(ctx) + if err != nil { + return nil, 0, err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +// isFull is a write a bucket or stream refused because it holds as much as it may. +func isFull(err error) bool { + msg := err.Error() + return strings.Contains(msg, "maximum bytes exceeded") || strings.Contains(msg, "maximum messages exceeded") || + strings.Contains(msg, "insufficient resources") +} + +func (s *busState) Write(key string, value []byte, revision uint64) (uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + kv, err := s.kv(ctx) + if err != nil { + return 0, err + } + var rev uint64 + if revision == 0 { + rev, err = kv.Create(ctx, key, value) + } else { + rev, err = kv.Update(ctx, key, value, revision) + } + switch { + case err == nil: + return rev, nil + case errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) || isWrongSequence(err): + return 0, ledger.ErrChanged + case isFull(err): + return 0, fmt.Errorf("%w: %v", ledger.ErrFull, err) + } + return 0, err +} + +func (s *busState) Keys() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + kv, err := s.kv(ctx) + if err != nil { + return nil, err + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +type busSnapshots struct { + read, write *Conn + prefix string +} + +func (s *busSnapshots) Get(key string) ([]byte, uint64, error) { + return lastOn(s.read, "", s.prefix+"."+key) +} + +func (s *busSnapshots) Put(key string, value []byte, expect uint64) error { + if s.write == nil { + return errors.New("a ledger's snapshot is published only on its module's ledger connection, and none is held") + } + _, err := publishExpecting(s.write, s.prefix+"."+key, value, expect) + return err +} diff --git a/node-tools/internal/ledger/ledger.go b/node-tools/internal/ledger/ledger.go new file mode 100644 index 0000000..e3b055b --- /dev/null +++ b/node-tools/internal/ledger/ledger.go @@ -0,0 +1,493 @@ +// Package ledger is the tool runner's side of a ledger (novox/hq ADR 0597, design 25 § 12.4; issue 1966): a storage type +// whose state is derived from its history. `apply` checks an operation against its type, appends it to the instance's +// history with the last sequence expected, derives the state by the operation's effect and writes it as an update of the +// revision read, and writes the instance's snapshot. What the history, the state and the snapshots are kept in is given +// (History, State, Snapshots): the bus in the runtime, fakes in a test. +package ledger + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The refusals the stores answer, told by errors.Is. +var ( + // ErrRaced is an append or a snapshot written with an expected sequence that is no longer the last: another writer + // wrote first. + ErrRaced = errors.New("another writer wrote the instance's history first") + // ErrChanged is a state written at a revision that is no longer the key's. + ErrChanged = errors.New("the state was written by another writer first") + // ErrFull is a state bucket that refuses a write because it is full. + ErrFull = errors.New("the state bucket is full") +) + +// CodeChanged is the error code a bundle is answered a lost race with, as a lost state update (mesh-sdk stdio.CodeChanged). +const CodeChanged = -32010 + +// raced is an apply refused because another writer appended to the instance first (ADR 0597 §5). +type raced struct{ instance string } + +func (e raced) Error() string { + return "the operation on " + e.instance + " is not taken: another writer appended to its history first; read it and apply again (novox/hq ADR 0597 §5)" +} +func (e raced) Is(target error) bool { return target == ErrRaced } +func (e raced) ErrorCode() int { return CodeChanged } + +// Kept is one message a history keeps: its sequence and its bytes. +type Kept struct { + Sequence uint64 + Data []byte +} + +// History is a ledger's history: a stream with a subject per instance, each instance named by its key. +type History interface { + // Last is the sequence of the instance's last operation, 0 when it has none. + Last(key string) (uint64, error) + // Append writes one operation, expecting the instance's last sequence to be expect (0: it has none); ErrRaced when + // it is not. + Append(key string, op []byte, expect uint64) (uint64, error) + // Read is the operations of the instances keys names (a key, with `*` for one name and `>` for the rest) from a + // sequence on, oldest first, at most limit of them (every one when limit is 0). + Read(keys string, from uint64, limit int) ([]Kept, error) +} + +// State is a ledger's state bucket: each instance's record under its key. +type State interface { + // Get is the key's value and revision; nil and 0 when it has none. + Get(key string) ([]byte, uint64, error) + // Write writes the key's value at the revision read (0: only when it has none); ErrChanged when the key is no longer + // at it, ErrFull when the bucket is full. + Write(key string, value []byte, revision uint64) (uint64, error) + // Keys is every key the bucket holds. + Keys() ([]string, error) +} + +// Snapshots are a ledger's snapshots: the latest record of each instance, on a subject of its own, never expiring. +type Snapshots interface { + // Get is the key's latest snapshot and the sequence it was written at; nil and 0 when it has none. + Get(key string) ([]byte, uint64, error) + // Put writes the key's snapshot, expecting its last to be at expect (0: none); ErrRaced when it is not. + Put(key string, value []byte, expect uint64) error +} + +// Ledger is one ledger of one module: its types and where it is kept. +type Ledger struct { + Module string + Name string + Types []catalogue.LedgerType + History History + State State + Snapshots Snapshots + // Now is the clock an operation is stamped by; time.Now when nil. + Now func() time.Time + // Logf says what an apply could not do after its operation was taken; nothing is said when nil. + Logf func(format string, args ...any) +} + +func (l *Ledger) say(format string, args ...any) { + if l.Logf != nil { + l.Logf(format, args...) + } +} + +// Record is what the state and a snapshot keep of an instance: its state and the sequence of the last operation in it. +type Record struct { + Sequence uint64 `json:"sequence"` + State json.RawMessage `json:"state"` +} + +// segment is one part of an instance's path. +var segment = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// KeyOf is the key an instance is kept under, in the state and as its history's and snapshot's subject token: its path's +// parts joined by dots (`decision/0290` is `decision.0290`). +func KeyOf(instance string) (string, error) { + parts := strings.Split(instance, "/") + if len(parts) < 2 { + return "", fmt.Errorf("%q is not an instance: an instance is named by a path whose first part is its type's name, "+ + "such as decision/0290 (novox/hq ADR 0597 §1)", instance) + } + for _, p := range parts { + if !segment.MatchString(p) { + return "", fmt.Errorf("%q is not an instance: each part of its path holds letters, digits, - and _ alone", instance) + } + } + return strings.Join(parts, "."), nil +} + +// InstanceOf is the instance a key names. +func InstanceOf(key string) string { return strings.ReplaceAll(key, ".", "/") } + +func (l *Ledger) now() time.Time { + if l.Now != nil { + return l.Now().UTC() + } + return time.Now().UTC() +} + +// typeOf is the instance's type, by its path's first part. +func (l *Ledger) typeOf(instance string) (catalogue.LedgerType, error) { + name, _, _ := strings.Cut(instance, "/") + for _, t := range l.Types { + if t.Name == name { + return t, nil + } + } + return catalogue.LedgerType{}, fmt.Errorf("%s's ledger %s has no type %q, which the instance %s names", l.Module, l.Name, name, instance) +} + +// operationOf is the operation a type allows by its name. +func operationOf(t catalogue.LedgerType, name string) (catalogue.LedgerOperation, error) { + for _, op := range t.Operations { + if op.Name == name { + return op, nil + } + } + return catalogue.LedgerOperation{}, fmt.Errorf("the type %s allows no operation %q", t.Name, name) +} + +// Digest is the digest a type pairs a text field with (ADR 0597 §2): sha256 of the text, as the knowledge base writes it. +func Digest(text string) string { + sum := sha256.Sum256([]byte(text)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// kinds is the JSON kind each schema type name is held to; a name not here is the judge's to know. +var kinds = map[string]func(any) bool{ + "string": func(v any) bool { _, ok := v.(string); return ok }, + "number": func(v any) bool { _, ok := v.(float64); return ok }, + "integer": func(v any) bool { f, ok := v.(float64); return ok && f == float64(int64(f)) }, + "boolean": func(v any) bool { _, ok := v.(bool); return ok }, + "object": func(v any) bool { _, ok := v.(map[string]any); return ok }, + "array": func(v any) bool { _, ok := v.([]any); return ok }, +} + +// checked is an operation as apply checked it: its instance's key, its operation and its body. +type checked struct { + key string + op catalogue.LedgerOperation + body map[string]any +} + +// check holds an operation to its type (ADR 0597 §2): a type the instance names, an operation it allows, a body that is an +// object whose fields fit the schema, the key its effect names, and every digest matching its text. +func (l *Ledger) check(r buscalls.ApplyRequest) (checked, error) { + key, err := KeyOf(r.Instance) + if err != nil { + return checked{}, err + } + t, err := l.typeOf(r.Instance) + if err != nil { + return checked{}, err + } + op, err := operationOf(t, r.Operation) + if err != nil { + return checked{}, err + } + body := map[string]any{} + if len(r.Body) > 0 && string(r.Body) != "null" { + if err := json.Unmarshal(r.Body, &body); err != nil { + return checked{}, fmt.Errorf("the operation %s's body is a JSON object: %w", r.Operation, err) + } + } + for name, field := range t.Schema { + v, has := body[name] + if !has { + continue + } + if fits, known := kinds[field.Type]; known && !fits(v) { + return checked{}, fmt.Errorf("the operation %s's field %s is a %s in the type %s", r.Operation, name, field.Type, t.Name) + } + } + for name, field := range t.Schema { + if field.DigestOf == "" { + continue + } + digest, hasDigest := body[name] + text, hasText := body[field.DigestOf] + if !hasDigest && !hasText { + continue + } + s, isText := text.(string) + if !hasDigest || !hasText || !isText || digest != Digest(s) { + return checked{}, fmt.Errorf("the operation %s's %s is not the digest of its %s, so it is refused (novox/hq ADR 0597 §2)", + r.Operation, name, field.DigestOf) + } + } + switch op.Effect { + case catalogue.EffectSetFields, catalogue.EffectAppend: + case catalogue.EffectSetEntry, catalogue.EffectRemove: + if op.Effect == catalogue.EffectRemove && op.Into == "" { + break + } + if _, ok := body[op.Key].(string); !ok { + return checked{}, fmt.Errorf("the operation %s names its entry by its field %s, a text, which its body lacks", r.Operation, op.Key) + } + default: + return checked{}, fmt.Errorf("the operation %s's effect %q is not one the mesh offers (novox/hq ADR 0597 §2)", r.Operation, op.Effect) + } + return checked{key: key, op: op, body: body}, nil +} + +// effect is an operation's effect on a state (ADR 0597 §2): it copies only the fields the operation names, and the +// operation's sequence and proven author where it says so; null is a removed instance. +func effect(state json.RawMessage, op catalogue.LedgerOperation, body map[string]any, seq uint64, author buscalls.Author) (json.RawMessage, error) { + s := map[string]any{} + if len(state) > 0 && string(state) != "null" { + if err := json.Unmarshal(state, &s); err != nil { + return nil, fmt.Errorf("the state held is not an object: %w", err) + } + } + copied := map[string]any{} + for _, f := range op.Fields { + if v, ok := body[f]; ok { + copied[f] = v + } + } + if op.Sequence { + copied["sequence"] = seq + } + // Written last, so a body's own `author` never stands for the proven one (ADR 0597 §7). + if op.Author { + copied["author"] = author + } + switch op.Effect { + case catalogue.EffectSetFields: + for k, v := range copied { + s[k] = v + } + case catalogue.EffectSetEntry: + m, _ := s[op.Into].(map[string]any) + if m == nil { + m = map[string]any{} + } + m[body[op.Key].(string)] = copied + s[op.Into] = m + case catalogue.EffectAppend: + list, _ := s[op.Into].([]any) + s[op.Into] = append(list, copied) + case catalogue.EffectRemove: + if op.Into == "" { + return json.RawMessage("null"), nil + } + if m, ok := s[op.Into].(map[string]any); ok { + delete(m, body[op.Key].(string)) + } + } + return json.Marshal(s) +} + +// fold applies one kept operation to a record, as apply derived it. +func (l *Ledger) fold(rec Record, k Kept) (Record, error) { + var o buscalls.Operation + if err := json.Unmarshal(k.Data, &o); err != nil { + return rec, fmt.Errorf("the history's operation %d is not one: %w", k.Sequence, err) + } + t, err := l.typeOf(o.Instance) + if err != nil { + return rec, err + } + op, err := operationOf(t, o.Operation) + if err != nil { + return rec, err + } + body := map[string]any{} + if len(o.Body) > 0 && string(o.Body) != "null" { + if err := json.Unmarshal(o.Body, &body); err != nil { + return rec, err + } + } + if (op.Effect == catalogue.EffectSetEntry || (op.Effect == catalogue.EffectRemove && op.Into != "")) && !isText(body[op.Key]) { + return rec, fmt.Errorf("the history's operation %d names no entry", k.Sequence) + } + state, err := effect(rec.State, op, body, k.Sequence, o.Author) + if err != nil { + return rec, err + } + return Record{Sequence: k.Sequence, State: state}, nil +} + +func isText(v any) bool { _, ok := v.(string); return ok } + +// replay folds the instance's operations after rec's sequence, up to upto (every one when 0), onto rec. +func (l *Ledger) replay(key string, rec Record, upto uint64) (Record, error) { + kept, err := l.History.Read(key, rec.Sequence+1, 0) + if err != nil { + return rec, err + } + for _, k := range kept { + if upto != 0 && k.Sequence > upto { + break + } + if k.Sequence <= rec.Sequence { + continue + } + if rec, err = l.fold(rec, k); err != nil { + return rec, err + } + } + return rec, nil +} + +// readState is the instance's record in the state and its revision; an empty record when it has none. +func (l *Ledger) readState(key string) (Record, uint64, error) { + raw, rev, err := l.State.Get(key) + if err != nil || raw == nil { + return Record{}, rev, err + } + var rec Record + if err := json.Unmarshal(raw, &rec); err != nil { + return Record{}, rev, fmt.Errorf("the state of %s is not a record: %w", key, err) + } + return rec, rev, nil +} + +// readSnapshot is the instance's snapshot and the sequence it was written at; an empty record when it has none. +func (l *Ledger) readSnapshot(key string) (Record, uint64, error) { + raw, at, err := l.Snapshots.Get(key) + if err != nil || raw == nil { + return Record{}, at, err + } + var rec Record + if err := json.Unmarshal(raw, &rec); err != nil { + return Record{}, at, fmt.Errorf("the snapshot of %s is not a record: %w", key, err) + } + return rec, at, nil +} + +// derive is the instance's record caught up with its history up to last: from the state, or from the snapshot when it +// is further on (an instance whose state was lost, or whose operations expired, ADR 0597 §8). +func (l *Ledger) derive(key string, base Record, last uint64) (Record, error) { + if base.Sequence >= last { + return base, nil + } + snap, _, err := l.readSnapshot(key) + if err != nil { + return base, err + } + if snap.Sequence > base.Sequence { + base = snap + } + return l.replay(key, base, last) +} + +// settle writes want into the state at the revision read (ADR 0597 §5): a lost update re-reads, and catches the state +// read up to want's sequence from the history, never appending again; a state already further on is left as it is. It +// answers what the state holds after it, and whether it is behind because the bucket is full (ADR 0597 §6). +func (l *Ledger) settle(key string, want Record, rev uint64) (Record, bool, error) { + for tries := 0; tries < 16; tries++ { + raw, err := json.Marshal(want) + if err != nil { + return want, false, err + } + _, err = l.State.Write(key, raw, rev) + switch { + case err == nil: + return want, false, nil + case errors.Is(err, ErrFull): + return want, true, nil + case !errors.Is(err, ErrChanged): + return want, false, err + } + cur, curRev, err := l.readState(key) + if err != nil { + return want, false, err + } + if cur.Sequence >= want.Sequence { + return cur, false, nil + } + if want, err = l.replay(key, cur, want.Sequence); err != nil { + return want, false, err + } + rev = curRev + } + return want, false, fmt.Errorf("the state of %s kept being written by other writers; the operation is in its history and the next apply catches it up", key) +} + +// writeSnapshot writes rec as the instance's snapshot unless the snapshot is already as far on (ADR 0597 §8). +func (l *Ledger) writeSnapshot(key string, rec Record) error { + for tries := 0; tries < 16; tries++ { + snap, at, err := l.readSnapshot(key) + if err != nil { + return err + } + if at != 0 && snap.Sequence >= rec.Sequence { + return nil + } + raw, err := json.Marshal(rec) + if err != nil { + return err + } + if err = l.Snapshots.Put(key, raw, at); err == nil || !errors.Is(err, ErrRaced) { + return err + } + } + return fmt.Errorf("the snapshot of %s kept being written by other writers", key) +} + +// Apply applies one operation to one instance, stamped with author (ADR 0597 §3, §5, §6, §8; design 25 § 12.4 steps 1-5): +// checked against its type, appended with the instance's last sequence expected, the state derived by its effect and +// written at the revision read, and the snapshot written. A racing writer's append is refused; a full state bucket +// answers taken with the state behind. +func (l *Ledger) Apply(r buscalls.ApplyRequest, author buscalls.Author) (buscalls.Applied, error) { + c, err := l.check(r) + if err != nil { + return buscalls.Applied{}, err + } + last, err := l.History.Last(c.key) + if err != nil { + return buscalls.Applied{}, err + } + held, rev, err := l.readState(c.key) + if err != nil { + return buscalls.Applied{}, err + } + base, err := l.derive(c.key, held, last) + if err != nil { + return buscalls.Applied{}, err + } + body := r.Body + if len(body) == 0 { + body = json.RawMessage("{}") + } + op, err := json.Marshal(buscalls.Operation{Instance: r.Instance, Operation: r.Operation, Body: body, Author: author, At: l.now()}) + if err != nil { + return buscalls.Applied{}, err + } + seq, err := l.History.Append(c.key, op, last) + if errors.Is(err, ErrRaced) { + return buscalls.Applied{}, raced{r.Instance} + } + if err != nil { + return buscalls.Applied{}, err + } + state, err := effect(base.State, c.op, c.body, seq, author) + if err != nil { + return buscalls.Applied{}, err + } + // From here the operation is taken: it is in the history, and a fault below is the state's to catch up, said here + // and answered as the state behind, never as a refusal of what later takes effect (ADR 0597 §6). + now, behind, err := l.settle(c.key, Record{Sequence: seq, State: state}, rev) + if err != nil { + l.say("[mesh-tools] %s's ledger %s took operation %d on %s and its state could not follow: %v; the next apply "+ + "or a rebuild catches it up", l.Module, l.Name, seq, r.Instance, err) + behind, now = true, Record{Sequence: seq, State: state} + } + if behind { + now = Record{Sequence: seq, State: state} + } + if err := l.writeSnapshot(c.key, now); err != nil { + l.say("[mesh-tools] %s's ledger %s took operation %d on %s and its snapshot could not be written: %v; the next "+ + "apply or a snapshot repair writes it", l.Module, l.Name, seq, r.Instance, err) + } + return buscalls.Applied{Taken: true, Sequence: seq, StateBehind: behind}, nil +} diff --git a/node-tools/internal/ledger/ledger_test.go b/node-tools/internal/ledger/ledger_test.go new file mode 100644 index 0000000..01b3017 --- /dev/null +++ b/node-tools/internal/ledger/ledger_test.go @@ -0,0 +1,102 @@ +package ledger + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +var owner = buscalls.Author{Account: "ledger.anchor.m"} + +// 1966/AC13, 1966/AC14 and the checks of an operation against its type (ADR 0597 §2, §5, §6, §8; design 25 § 12.4). +func TestApplyAppendsDerivesAndSnapshots(t *testing.T) { + cases := []struct { + name string + run func(t *testing.T, l *Ledger, f fakes) + }{ + {"1966/AC13 an instance at sequence 4 is taken with 5, its state and snapshot the effect's result", func(t *testing.T, l *Ledger, f fakes) { + for i := 1; i <= 4; i++ { + applied(t, l, "decision/0290", "note", fmt.Sprintf(`{"says":"n%d"}`, i), owner) + } + if last, _ := f.history.Last("decision.0290"); last != 4 { + t.Fatalf("the instance is at %d, want 4", last) + } + got := applied(t, l, "decision/0290", "title", `{"title":"A ledger","ignored":"x"}`, owner) + if !got.Taken || got.Sequence != 5 || got.StateBehind { + t.Fatalf("answered %+v, want taken with 5", got) + } + raw, _, _ := f.state.Get("decision.0290") + state := record(t, raw) + want := `{"author":{"account":"ledger.anchor.m"},"notes":[{"says":"n1"},{"says":"n2"},{"says":"n3"},{"says":"n4"}],"sequence":5,"title":"A ledger"}` + if state.Sequence != 5 || string(state.State) != want { + t.Fatalf("the state is %d %s, want 5 %s", state.Sequence, state.State, want) + } + snap, _, _ := f.snapshots.Get("decision.0290") + if string(snap) != string(raw) { + t.Fatalf("the snapshot is %s, the state %s", snap, raw) + } + }}, + {"1966/AC14 a full state bucket answers taken and state behind, and the next apply after room catches up", func(t *testing.T, l *Ledger, f fakes) { + applied(t, l, "decision/0290", "note", `{"says":"first"}`, owner) + f.state.Full = true + got := applied(t, l, "decision/0290", "note", `{"says":"while full"}`, owner) + if !got.Taken || got.Sequence != 2 || !got.StateBehind { + t.Fatalf("answered %+v, want taken with 2 and state behind", got) + } + if raw, _, _ := f.state.Get("decision.0290"); record(t, raw).Sequence != 1 { + t.Fatalf("the full state moved: %s", raw) + } + f.state.Full = false + got = applied(t, l, "decision/0290", "note", `{"says":"after room"}`, owner) + raw, _, _ := f.state.Get("decision.0290") + state := record(t, raw) + if got.StateBehind || state.Sequence != 3 || string(state.State) != `{"notes":[{"says":"first"},{"says":"while full"},{"says":"after room"}]}` { + t.Fatalf("after room: %+v, the state %d %s", got, state.Sequence, state.State) + } + }}, + {"each effect: set an entry, remove it, remove the instance", func(t *testing.T, l *Ledger, f fakes) { + text := "the section's text" + applied(t, l, "decision/1", "section", fmt.Sprintf(`{"id":"a","text":%q,"digest":%q}`, text, Digest(text)), owner) + applied(t, l, "decision/1", "section", fmt.Sprintf(`{"id":"b","text":"b","digest":%q}`, Digest("b")), owner) + applied(t, l, "decision/1", "drop-section", `{"id":"a"}`, owner) + raw, _, _ := f.state.Get("decision.1") + if got := string(record(t, raw).State); got != fmt.Sprintf(`{"sections":{"b":{"digest":%q}}}`, Digest("b")) { + t.Fatalf("the state is %s", got) + } + applied(t, l, "decision/1", "withdraw", `{}`, owner) + raw, _, _ = f.state.Get("decision.1") + if got := record(t, raw); got.Sequence != 4 || string(got.State) != "null" { + t.Fatalf("the removed instance is %d %s", got.Sequence, got.State) + } + }}, + {"an operation outside its type, a body off its schema and a digest off its text are refused", func(t *testing.T, l *Ledger, f fakes) { + for _, c := range []struct{ instance, operation, body, refusal string }{ + {"listing/a17", "title", `{}`, `no type "listing"`}, + {"decision/0290", "accept", `{}`, `no operation "accept"`}, + {"decision/0290", "title", `{"title":7}`, "field title is a string"}, + {"decision/0290", "section", `{"id":"a","text":"x","digest":"sha256:00"}`, "is not the digest of its text"}, + {"decision/0290", "section", `{"text":"x"}`, "is not the digest"}, + {"decision/0290", "drop-section", `{}`, "names its entry by its field id"}, + {"decision", "title", `{}`, "is not an instance"}, + {"decision/a.b", "title", `{}`, "is not an instance"}, + } { + _, err := l.Apply(buscalls.ApplyRequest{Instance: c.instance, Operation: c.operation, Body: json.RawMessage(c.body)}, owner) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s %s %s: %v, want %q", c.instance, c.operation, c.body, err, c.refusal) + } + } + if last, _ := f.history.Last("decision.0290"); last != 0 { + t.Fatalf("a refused operation was appended: %d", last) + } + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + l, f := newLedger() + c.run(t, l, f) + }) + } +} diff --git a/node-tools/internal/ledger/memory.go b/node-tools/internal/ledger/memory.go new file mode 100644 index 0000000..c038888 --- /dev/null +++ b/node-tools/internal/ledger/memory.go @@ -0,0 +1,158 @@ +package ledger + +import ( + "sort" + "strings" + "sync" +) + +// The memory stores keep what the bus keeps, as the bus answers it, for a test of a ledger without a bus: a stream whose +// sequence counts across every instance, a bucket with revisions that may be full, and snapshots written with an expected +// sequence. + +// MemoryHistory is a history in memory. +type MemoryHistory struct { + mu sync.Mutex + seq uint64 + kept []struct { + key string + Kept + } +} + +// Matches is whether a key pattern (`*` one name, `>` the rest) names a key. +func Matches(pattern, key string) bool { + p, k := strings.Split(pattern, "."), strings.Split(key, ".") + for i, part := range p { + if part == ">" { + return len(k) > i + } + if i >= len(k) || (part != "*" && part != k[i]) { + return false + } + } + return len(p) == len(k) +} + +func (h *MemoryHistory) Last(key string) (uint64, error) { + h.mu.Lock() + defer h.mu.Unlock() + last := uint64(0) + for _, k := range h.kept { + if k.key == key { + last = k.Sequence + } + } + return last, nil +} + +func (h *MemoryHistory) Append(key string, op []byte, expect uint64) (uint64, error) { + last, _ := h.Last(key) + h.mu.Lock() + defer h.mu.Unlock() + if last != expect { + return 0, ErrRaced + } + h.seq++ + h.kept = append(h.kept, struct { + key string + Kept + }{key, Kept{Sequence: h.seq, Data: op}}) + return h.seq, nil +} + +func (h *MemoryHistory) Read(keys string, from uint64, limit int) ([]Kept, error) { + h.mu.Lock() + defer h.mu.Unlock() + var out []Kept + for _, k := range h.kept { + if k.Sequence >= from && Matches(keys, k.key) && (limit == 0 || len(out) < limit) { + out = append(out, k.Kept) + } + } + return out, nil +} + +// Expire drops every operation of the history, as a ledger's keeping days end them. +func (h *MemoryHistory) Expire() { + h.mu.Lock() + defer h.mu.Unlock() + h.kept = nil +} + +type memoryValue struct { + value []byte + rev uint64 +} + +// MemoryState is a state bucket in memory. +type MemoryState struct { + mu sync.Mutex + rev uint64 + // Full makes every write refused as a full bucket's. + Full bool + kv map[string]memoryValue +} + +func (s *MemoryState) Get(key string) ([]byte, uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + v := s.kv[key] + return v.value, v.rev, nil +} + +func (s *MemoryState) Write(key string, value []byte, revision uint64) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.Full { + return 0, ErrFull + } + if s.kv[key].rev != revision { + return 0, ErrChanged + } + if s.kv == nil { + s.kv = map[string]memoryValue{} + } + s.rev++ + s.kv[key] = memoryValue{value, s.rev} + return s.rev, nil +} + +func (s *MemoryState) Keys() ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.kv { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// MemorySnapshots are snapshots in memory. +type MemorySnapshots struct { + mu sync.Mutex + at uint64 + kv map[string]memoryValue +} + +func (s *MemorySnapshots) Get(key string) ([]byte, uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + v := s.kv[key] + return v.value, v.rev, nil +} + +func (s *MemorySnapshots) Put(key string, value []byte, expect uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[key].rev != expect { + return ErrRaced + } + if s.kv == nil { + s.kv = map[string]memoryValue{} + } + s.at++ + s.kv[key] = memoryValue{value, s.at} + return nil +} diff --git a/node-tools/internal/ledger/types_test.go b/node-tools/internal/ledger/types_test.go new file mode 100644 index 0000000..7d8908a --- /dev/null +++ b/node-tools/internal/ledger/types_test.go @@ -0,0 +1,60 @@ +package ledger + +import ( + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// records is a ledger type of each effect: a title set, a section set as an entry, a note appended, a section dropped, +// and the whole instance removed. +var records = catalogue.LedgerType{ + Name: "decision", + Schema: map[string]catalogue.LedgerField{ + "title": {Type: "string"}, "text": {Type: "string"}, "digest": {Type: "string", DigestOf: "text"}, + }, + Operations: []catalogue.LedgerOperation{ + {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Sequence: true, Author: true}, + {Name: "section", Effect: catalogue.EffectSetEntry, Into: "sections", Key: "id", Fields: []string{"digest"}}, + {Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}}, + {Name: "drop-section", Effect: catalogue.EffectRemove, Into: "sections", Key: "id"}, + {Name: "withdraw", Effect: catalogue.EffectRemove}, + }, + Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}, +} + +type fakes struct { + history *MemoryHistory + state *MemoryState + snapshots *MemorySnapshots +} + +func newLedger() (*Ledger, fakes) { + f := fakes{&MemoryHistory{}, &MemoryState{}, &MemorySnapshots{}} + return &Ledger{Module: "m", Name: "records", Types: []catalogue.LedgerType{records}, History: f.history, State: f.state, + Snapshots: f.snapshots}, f +} + +// applied applies an operation and fails the test on a refusal. +func applied(t *testing.T, l *Ledger, instance, operation, body string, author buscalls.Author) buscalls.Applied { + t.Helper() + got, err := l.Apply(buscalls.ApplyRequest{Ledger: l.Name, Instance: instance, Operation: operation, Body: json.RawMessage(body)}, author) + if err != nil { + t.Fatalf("%s %s: %v", operation, body, err) + } + return got +} + +// record is the instance's record a store holds, read as a Record. +func record(t *testing.T, raw []byte) Record { + t.Helper() + var r Record + if raw != nil { + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + } + return r +} -- 2.54.0 From 63c1e3faa27dfc5d8ea113eef20ab6f41d2961ab Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:42:33 +0100 Subject: [PATCH 03/12] Read a ledger: an instance's state, its history oldest first with sequence and author, a watch of its states, a declared index (issue 1966 T3) --- node-tools/internal/bus/ledger.go | 114 +++++++++++++++++ node-tools/internal/ledger/ledger.go | 2 + node-tools/internal/ledger/memory.go | 49 +++++++ node-tools/internal/ledger/read.go | 155 +++++++++++++++++++++++ node-tools/internal/ledger/read_test.go | 67 ++++++++++ node-tools/internal/ledger/types_test.go | 5 +- 6 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 node-tools/internal/ledger/read.go create mode 100644 node-tools/internal/ledger/read_test.go diff --git a/node-tools/internal/bus/ledger.go b/node-tools/internal/bus/ledger.go index bf46d53..5591379 100644 --- a/node-tools/internal/bus/ledger.go +++ b/node-tools/internal/bus/ledger.go @@ -5,9 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strconv" "strings" + "time" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -227,3 +230,114 @@ func (s *busSnapshots) Put(key string, value []byte, expect uint64) error { _, err := publishExpecting(s.write, s.prefix+"."+key, value, expect) return err } + +// Watch hands deliver every instance's record under a key pattern as it is, then each change; a removed key is handed as +// nil. +func (s *busState) Watch(pattern string, deliver func(key string, value []byte, current bool) error) (func(), error) { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + kv, err := s.kv(ctx) + cancel() + if err != nil { + return nil, err + } + w, err := kv.Watch(context.Background(), pattern) + if err != nil { + return nil, err + } + hand := func(e jetstream.KeyValueEntry, current bool) { + var value []byte + if e.Operation() == jetstream.KeyValuePut { + value = e.Value() + } + if err := deliver(e.Key(), value, current); err != nil { + s.c.Logf("[mesh-tools] a ledger watch of %s was not taken: %v", s.bucket, err) + } + } + timer := time.NewTimer(StateTimeout) + defer timer.Stop() + for current := true; current; { + select { + case e, ok := <-w.Updates(): + if !ok { + return nil, errors.New("the ledger's state watch ended before its current states were handed over") + } + if e == nil { + current = false + continue + } + hand(e, true) + case <-timer.C: + _ = w.Stop() + return nil, fmt.Errorf("the ledger's state %s did not hand its current states in %s", s.bucket, StateTimeout) + } + } + go func() { + for e := range w.Updates() { + if e != nil { + hand(e, false) + } + } + }() + return func() { _ = w.Stop() }, nil +} + +// LedgerIndexes are the buckets of a ledger binding's declared indexes, by index name, on the runner's own connection. +func (c *Conn) LedgerIndexes(b membership.BusBinding) map[string]ledger.Index { + out := map[string]ledger.Index{} + for name, bucket := range b.Indexes { + out[name] = &busState{c: c, bucket: bucket} + } + return out +} + +func (s *busState) Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) { + if pattern == "" { + pattern = ">" + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + var out []buscalls.IndexEntry + for _, k := range keys { + if limit > 0 && len(out) >= limit { + break + } + if !SubjectMatches(pattern, k) { + continue + } + v, _, err := s.Get(k) + if err != nil { + return out, err + } + if v != nil { + out = append(out, buscalls.IndexEntry{Key: k, Value: valueOf(v)}) + } + } + return out, nil +} + +func (s *busState) Put(key string, value []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + kv, err := s.kv(ctx) + if err != nil { + return err + } + _, err = kv.Put(ctx, key, value) + return err +} + +func (s *busState) Delete(key string) error { + ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) + defer cancel() + kv, err := s.kv(ctx) + if err != nil { + return err + } + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} diff --git a/node-tools/internal/ledger/ledger.go b/node-tools/internal/ledger/ledger.go index e3b055b..f39b889 100644 --- a/node-tools/internal/ledger/ledger.go +++ b/node-tools/internal/ledger/ledger.go @@ -87,6 +87,8 @@ type Ledger struct { History History State State Snapshots Snapshots + // Indexes are its declared indexes' buckets, by index name. + Indexes map[string]Index // Now is the clock an operation is stamped by; time.Now when nil. Now func() time.Time // Logf says what an apply could not do after its operation was taken; nothing is said when nil. diff --git a/node-tools/internal/ledger/memory.go b/node-tools/internal/ledger/memory.go index c038888..240a225 100644 --- a/node-tools/internal/ledger/memory.go +++ b/node-tools/internal/ledger/memory.go @@ -1,9 +1,12 @@ package ledger import ( + "encoding/json" "sort" "strings" "sync" + + "git.novox.be/novox/mesh-sdk/go/buscalls" ) // The memory stores keep what the bus keeps, as the bus answers it, for a test of a ledger without a bus: a stream whose @@ -156,3 +159,49 @@ func (s *MemorySnapshots) Put(key string, value []byte, expect uint64) error { s.kv[key] = memoryValue{value, s.at} return nil } + +// MemoryIndex is an index's bucket in memory. +type MemoryIndex struct { + mu sync.Mutex + kv map[string][]byte +} + +func (x *MemoryIndex) Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) { + x.mu.Lock() + defer x.mu.Unlock() + if pattern == "" { + pattern = ">" + } + var keys []string + for k := range x.kv { + if Matches(pattern, k) { + keys = append(keys, k) + } + } + sort.Strings(keys) + var out []buscalls.IndexEntry + for _, k := range keys { + if limit > 0 && len(out) >= limit { + break + } + out = append(out, buscalls.IndexEntry{Key: k, Value: json.RawMessage(x.kv[k])}) + } + return out, nil +} + +func (x *MemoryIndex) Put(key string, value []byte) error { + x.mu.Lock() + defer x.mu.Unlock() + if x.kv == nil { + x.kv = map[string][]byte{} + } + x.kv[key] = value + return nil +} + +func (x *MemoryIndex) Delete(key string) error { + x.mu.Lock() + defer x.mu.Unlock() + delete(x.kv, key) + return nil +} diff --git a/node-tools/internal/ledger/read.go b/node-tools/internal/ledger/read.go new file mode 100644 index 0000000..08c2e1c --- /dev/null +++ b/node-tools/internal/ledger/read.go @@ -0,0 +1,155 @@ +package ledger + +import ( + "encoding/json" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// Reading a ledger (ADR 0597 §10, design 25 § 12.5; issue 1966 T3): an instance's state, its history, a watch of its +// states, and a declared index. Only the owning module reads its ledger: the runtime opens a ledger for the module whose +// bundle asks, never one the request names. + +// Index is one declared index's bucket, kept by a watcher in the owning module's process and rebuilt by reindex (ADR +// 0597 §4). +type Index interface { + // Entries is the entries whose key a pattern names (`*` one name, `>` the rest; empty is every key), at most limit + // (every one when 0). + Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) + // Put writes one entry; Delete removes one. + Put(key string, value []byte) error + Delete(key string) error +} + +// Watcher is a state bucket that hands each instance's record as it is and as it changes, under a key pattern. +type Watcher interface { + Watch(pattern string, deliver func(key string, value []byte, current bool) error) (func(), error) +} + +// HistoryMost is the most operations one history read answers. +const HistoryMost = 1000 + +// Get is an instance's state as the state holds it, and the sequence of its last operation in it; null and 0 for an +// instance with none. +func (l *Ledger) Get(r buscalls.InstanceRequest) (buscalls.InstanceState, error) { + key, err := KeyOf(r.Instance) + if err != nil { + return buscalls.InstanceState{}, err + } + rec, _, err := l.readState(key) + if err != nil { + return buscalls.InstanceState{}, err + } + state := rec.State + if len(state) == 0 { + state = json.RawMessage("null") + } + return buscalls.InstanceState{Instance: r.Instance, State: state, Sequence: rec.Sequence}, nil +} + +// operationsOf reads kept operations as the history answers them, each with its sequence. +func operationsOf(kept []Kept) ([]buscalls.Operation, error) { + out := make([]buscalls.Operation, 0, len(kept)) + for _, k := range kept { + var o buscalls.Operation + if err := json.Unmarshal(k.Data, &o); err != nil { + return nil, fmt.Errorf("the history's operation %d is not one: %w", k.Sequence, err) + } + o.Sequence = k.Sequence + out = append(out, o) + } + return out, nil +} + +// ReadHistory is an instance's operations from a sequence on, oldest first, each with its sequence and its proven author. +func (l *Ledger) ReadHistory(r buscalls.HistoryRequest) (buscalls.HistoryPage, error) { + key, err := KeyOf(r.Instance) + if err != nil { + return buscalls.HistoryPage{}, err + } + limit := r.Limit + if limit <= 0 || limit > HistoryMost { + limit = HistoryMost + } + last, err := l.History.Last(key) + if err != nil { + return buscalls.HistoryPage{}, err + } + kept, err := l.History.Read(key, r.From, limit+1) + if err != nil { + return buscalls.HistoryPage{}, err + } + page := buscalls.HistoryPage{Last: last} + if len(kept) > limit { + page.Next = kept[limit].Sequence + kept = kept[:limit] + } + if page.Operations, err = operationsOf(kept); err != nil { + return buscalls.HistoryPage{}, err + } + return page, nil +} + +// Watch hands deliver the state of every instance Instances names (`*` one name, `>` the rest, in an instance's path), +// the current ones first, then each change. +func (l *Ledger) Watch(r buscalls.LedgerWatchRequest, deliver func(buscalls.InstanceChange) error) (func(), error) { + w, ok := l.State.(Watcher) + if !ok { + return nil, errors.New("this ledger's state cannot be watched") + } + pattern := ">" + if r.Instances != "" { + pattern = keyPattern(r.Instances) + } + return w.Watch(pattern, func(key string, value []byte, current bool) error { + var rec Record + if value != nil { + if err := json.Unmarshal(value, &rec); err != nil { + return fmt.Errorf("the state of %s is not a record: %w", key, err) + } + } + state := rec.State + if len(state) == 0 { + state = json.RawMessage("null") + } + return deliver(buscalls.InstanceChange{Watch: r.Watch, Instance: InstanceOf(key), State: state, Sequence: rec.Sequence, + Current: current}) + }) +} + +// keyPattern is an instance pattern as a key pattern: its path's parts joined by dots. +func keyPattern(instances string) string { + out := []byte(instances) + for i, c := range out { + if c == '/' { + out[i] = '.' + } + } + return string(out) +} + +// index is a declared index by its name, or the reason there is none. +func (l *Ledger) index(name string) (Index, error) { + if ix, ok := l.Indexes[name]; ok { + return ix, nil + } + return nil, fmt.Errorf("%s's ledger %s declares no index %q", l.Module, l.Name, name) +} + +// Query is a declared index's entries whose key Key names. +func (l *Ledger) Query(r buscalls.QueryRequest) (buscalls.QueryAnswer, error) { + ix, err := l.index(r.Index) + if err != nil { + return buscalls.QueryAnswer{}, err + } + entries, err := ix.Entries(r.Key, r.Limit) + if err != nil { + return buscalls.QueryAnswer{}, err + } + if entries == nil { + entries = []buscalls.IndexEntry{} + } + return buscalls.QueryAnswer{Entries: entries}, nil +} diff --git a/node-tools/internal/ledger/read_test.go b/node-tools/internal/ledger/read_test.go new file mode 100644 index 0000000..17584db --- /dev/null +++ b/node-tools/internal/ledger/read_test.go @@ -0,0 +1,67 @@ +package ledger + +import ( + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// 1966/AC15 and the other reads of a ledger: get, query (ADR 0597 §10; design 25 § 12.5). +func TestReadsOfALedger(t *testing.T) { + cases := []struct { + name string + run func(t *testing.T, l *Ledger, f fakes) + }{ + {"1966/AC15 history answers an instance's operations oldest first with sequence and author", func(t *testing.T, l *Ledger, f fakes) { + op := buscalls.Author{Account: "ledger.anchor.m", Caller: "op@shanks"} + applied(t, l, "decision/1", "note", `{"says":"one"}`, owner) + applied(t, l, "decision/2", "note", `{"says":"elsewhere"}`, owner) + applied(t, l, "decision/1", "title", `{"title":"Two"}`, op) + applied(t, l, "decision/1", "note", `{"says":"three"}`, owner) + page, err := l.ReadHistory(buscalls.HistoryRequest{Instance: "decision/1", Limit: 2}) + if err != nil { + t.Fatal(err) + } + var got []string + for _, o := range page.Operations { + got = append(got, fmt.Sprintf("%s@%d %s %s", o.Operation, o.Sequence, o.Author.Account, o.Author.Caller)) + } + if strings.Join(got, "; ") != "note@1 ledger.anchor.m ; title@3 ledger.anchor.m op@shanks" || page.Next != 4 || page.Last != 4 { + t.Fatalf("the history is %q, next %d, last %d", got, page.Next, page.Last) + } + rest, _ := l.ReadHistory(buscalls.HistoryRequest{Instance: "decision/1", From: page.Next}) + if len(rest.Operations) != 1 || rest.Operations[0].Sequence != 4 || rest.Next != 0 { + t.Fatalf("read on: %+v", rest) + } + }}, + {"get answers the state and its sequence, null for an instance with none", func(t *testing.T, l *Ledger, f fakes) { + applied(t, l, "decision/1", "title", `{"title":"One"}`, owner) + got, err := l.Get(buscalls.InstanceRequest{Instance: "decision/1"}) + if err != nil || got.Sequence != 1 || !strings.Contains(string(got.State), `"title":"One"`) { + t.Fatalf("got %+v, %v", got, err) + } + if none, err := l.Get(buscalls.InstanceRequest{Instance: "decision/9"}); err != nil || string(none.State) != "null" { + t.Fatalf("none is %+v, %v", none, err) + } + }}, + {"query reads a declared index by key, and an undeclared one is refused", func(t *testing.T, l *Ledger, f fakes) { + _ = f.index.Put("One.decision.1", []byte(`{"instance":"decision/1"}`)) + _ = f.index.Put("Two.decision.2", []byte(`{"instance":"decision/2"}`)) + got, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "One.>"}) + if err != nil || len(got.Entries) != 1 || string(got.Entries[0].Value) != `{"instance":"decision/1"}` { + t.Fatalf("queried %+v, %v", got, err) + } + if _, err := l.Query(buscalls.QueryRequest{Index: "by-date"}); err == nil || !strings.Contains(err.Error(), `no index "by-date"`) { + t.Fatalf("an undeclared index: %v", err) + } + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + l, f := newLedger() + c.run(t, l, f) + }) + } +} diff --git a/node-tools/internal/ledger/types_test.go b/node-tools/internal/ledger/types_test.go index 7d8908a..1889063 100644 --- a/node-tools/internal/ledger/types_test.go +++ b/node-tools/internal/ledger/types_test.go @@ -29,12 +29,13 @@ type fakes struct { history *MemoryHistory state *MemoryState snapshots *MemorySnapshots + index *MemoryIndex } func newLedger() (*Ledger, fakes) { - f := fakes{&MemoryHistory{}, &MemoryState{}, &MemorySnapshots{}} + f := fakes{&MemoryHistory{}, &MemoryState{}, &MemorySnapshots{}, &MemoryIndex{}} return &Ledger{Module: "m", Name: "records", Types: []catalogue.LedgerType{records}, History: f.history, State: f.state, - Snapshots: f.snapshots}, f + Snapshots: f.snapshots, Indexes: map[string]Index{"by-title": f.index}}, f } // applied applies an operation and fails the test on a refusal. -- 2.54.0 From 8fe7ef3b82dd0b37ea3609b3f62f9d388893729b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:46:54 +0100 Subject: [PATCH 04/12] Serve a ledger owner its repairs rebuild, reindex, snapshot and lag as tools, its own ledgers only, marked acts or reads (issue 1966 T4) --- node-tools/internal/launch/launch.go | 3 + node-tools/internal/ledger/read_test.go | 57 +++- node-tools/internal/ledger/repair.go | 315 ++++++++++++++++++++ node-tools/internal/runtime/ledgers.go | 146 +++++++++ node-tools/internal/runtime/ledgers_test.go | 74 +++++ node-tools/internal/runtime/runtime.go | 64 +++- 6 files changed, 656 insertions(+), 3 deletions(-) create mode 100644 node-tools/internal/ledger/repair.go create mode 100644 node-tools/internal/runtime/ledgers.go create mode 100644 node-tools/internal/runtime/ledgers_test.go diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 0b91f71..17d95d4 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -39,6 +39,9 @@ type Tool struct { Name string Description string Input json.RawMessage + // Effect is what the tool does, reads, acts or dangerous (novox/hq ADR 0387 §1), where the runtime knows it: its own + // tools say it; a bundle's tool is marked by its manifest. + Effect string // Run calls the tool for caller: the bus user the call's subject named, or "" when it named none. A // caller is handed to the bundle in the call's `_meta` (MetaCaller), never among its arguments. Run func(caller string, args json.RawMessage) (json.RawMessage, error) diff --git a/node-tools/internal/ledger/read_test.go b/node-tools/internal/ledger/read_test.go index 17584db..2539c63 100644 --- a/node-tools/internal/ledger/read_test.go +++ b/node-tools/internal/ledger/read_test.go @@ -8,8 +8,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscalls" ) -// 1966/AC15 and the other reads of a ledger: get, query (ADR 0597 §10; design 25 § 12.5). -func TestReadsOfALedger(t *testing.T) { +// 1966/AC15, the other reads of a ledger (get, query) and its repairs (ADR 0597 §8, §10; design 25 § 12.5). +func TestReadsAndRepairsOfALedger(t *testing.T) { cases := []struct { name string run func(t *testing.T, l *Ledger, f fakes) @@ -57,6 +57,59 @@ func TestReadsOfALedger(t *testing.T) { t.Fatalf("an undeclared index: %v", err) } }}, + {"a lost state rebuilds from the history, an expired one from its snapshot, and lag says what is behind", func(t *testing.T, l *Ledger, f fakes) { + applied(t, l, "decision/1", "title", `{"title":"One"}`, owner) + applied(t, l, "decision/1", "note", `{"says":"kept"}`, owner) + want, _, _ := f.state.Get("decision.1") + f.state.kv = nil + lag, _ := l.Lag(buscalls.LedgerRequest{}) + if len(lag.Instances) != 1 || lag.Instances[0].History != 2 || lag.Instances[0].State != 0 || lag.Instances[0].Snapshot != 2 { + t.Fatalf("lag says %+v", lag) + } + if got, err := l.Rebuild(buscalls.RebuildRequest{}); err != nil || got.Written != 1 { + t.Fatalf("rebuilt %+v, %v", got, err) + } + if got, _, _ := f.state.Get("decision.1"); string(record(t, got).State) != string(record(t, want).State) { + t.Fatalf("rebuilt from the history %s, want %s", got, want) + } + f.state.kv = nil + f.history.Expire() + if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/1", From: buscalls.RebuildFromSnapshot}); err != nil || got.Written != 1 { + t.Fatalf("rebuilt %+v, %v", got, err) + } + if got, _, _ := f.state.Get("decision.1"); string(got) != string(want) { + t.Fatalf("rebuilt from the snapshot %s, want %s", got, want) + } + }}, + {"reindex builds a declared index from the states, and lag says an index behind until it does", func(t *testing.T, l *Ledger, f fakes) { + applied(t, l, "decision/1", "title", `{"title":"One ledger"}`, owner) + applied(t, l, "decision/2", "note", `{"says":"untitled"}`, owner) + _ = f.index.Put("stale.decision.9", []byte(`{}`)) + if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 1 || lag.Indexes[0].Behind != 1 { + t.Fatalf("lag says %+v", lag) + } + if got, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil || got.Written != 1 { + t.Fatalf("reindexed %+v, %v", got, err) + } + entries, _ := f.index.Entries("", 0) + if len(entries) != 1 || entries[0].Key != "One_ledger.decision.1" || string(entries[0].Value) != `{"instance":"decision/1","sequence":1}` { + t.Fatalf("the index holds %+v", entries) + } + if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 0 || len(lag.Instances) != 0 { + t.Fatalf("lag after reindex says %+v", lag) + } + }}, + {"snapshot rewrites a snapshot behind its state", func(t *testing.T, l *Ledger, f fakes) { + applied(t, l, "decision/1", "title", `{"title":"One"}`, owner) + f.snapshots.kv = nil + if got, err := l.Snapshot(buscalls.InstanceRequest{}); err != nil || got.Written != 1 { + t.Fatalf("snapshot %+v, %v", got, err) + } + state, _, _ := f.state.Get("decision.1") + if snap, _, _ := f.snapshots.Get("decision.1"); string(snap) != string(state) { + t.Fatalf("the snapshot %s, the state %s", snap, state) + } + }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/node-tools/internal/ledger/repair.go b/node-tools/internal/ledger/repair.go new file mode 100644 index 0000000..e0d2803 --- /dev/null +++ b/node-tools/internal/ledger/repair.go @@ -0,0 +1,315 @@ +package ledger + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// The ledger's repairs (ADR 0597 §8, §10; design 25 § 12.5; issue 1966 T4): rebuild an instance's state from its history +// or its snapshot, rewrite a snapshot behind its state, build a declared index again, and say what is behind its history. +// The tool runner serves them as tools of every module owning a ledger, its own ledgers only (the runtime's repairs.go). + +// instances is every instance the ledger knows, by key: those its state holds and those its history names. +func (l *Ledger) instances() ([]string, error) { + seen := map[string]bool{} + keys, err := l.State.Keys() + if err != nil { + return nil, err + } + for _, k := range keys { + seen[k] = true + } + kept, err := l.History.Read(">", 1, 0) + if err != nil { + return nil, err + } + ops, err := operationsOf(kept) + if err != nil { + return nil, err + } + for _, o := range ops { + if k, err := KeyOf(o.Instance); err == nil { + seen[k] = true + } + } + out := make([]string, 0, len(seen)) + for k := range seen { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// keysOf is the one instance a request names, or every instance when it names none. +func (l *Ledger) keysOf(instance string) ([]string, error) { + if instance == "" { + return l.instances() + } + k, err := KeyOf(instance) + if err != nil { + return nil, err + } + return []string{k}, nil +} + +// Rebuild derives the state of an instance (every instance when it names none) again, from its whole history or from its +// snapshot and the operations after it, and writes it, and its snapshot when that is behind. A derivation further back +// than the state holds is not written: the state never goes back (ADR 0597 §5). +func (l *Ledger) Rebuild(r buscalls.RebuildRequest) (buscalls.Repaired, error) { + from := r.From + if from == "" { + from = buscalls.RebuildFromHistory + } + if from != buscalls.RebuildFromHistory && from != buscalls.RebuildFromSnapshot { + return buscalls.Repaired{}, fmt.Errorf("a rebuild is from %q or %q, not %q", buscalls.RebuildFromHistory, buscalls.RebuildFromSnapshot, from) + } + keys, err := l.keysOf(r.Instance) + if err != nil { + return buscalls.Repaired{}, err + } + var out buscalls.Repaired + for _, key := range keys { + base := Record{} + if from == buscalls.RebuildFromSnapshot { + if base, _, err = l.readSnapshot(key); err != nil { + return out, err + } + } + derived, err := l.replay(key, base, 0) + if err != nil { + return out, err + } + held, rev, err := l.readState(key) + if err != nil { + return out, err + } + if derived.Sequence < held.Sequence { + continue + } + now, _, err := l.settle(key, derived, rev) + if err != nil { + return out, err + } + if err := l.writeSnapshot(key, now); err != nil { + return out, err + } + out.Written++ + } + return out, nil +} + +// Snapshot rewrites the snapshot of an instance (every instance when it names none) that is behind its state. +func (l *Ledger) Snapshot(r buscalls.InstanceRequest) (buscalls.Repaired, error) { + keys, err := l.keysOf(r.Instance) + if err != nil { + return buscalls.Repaired{}, err + } + var out buscalls.Repaired + for _, key := range keys { + held, _, err := l.readState(key) + if err != nil { + return out, err + } + snap, at, err := l.readSnapshot(key) + if err != nil { + return out, err + } + if held.Sequence == 0 || (at != 0 && snap.Sequence >= held.Sequence) { + continue + } + if err := l.writeSnapshot(key, held); err != nil { + return out, err + } + out.Written++ + } + return out, nil +} + +// unsafe is what a key part cannot hold. +var unsafe = regexp.MustCompile(`[^-_A-Za-z0-9]`) + +// IndexEntryOf is an instance's entry in a declared index, as reindex writes it: its key the values of the fields the +// index is on, in their order, then the instance's key, each part with what a key cannot hold written as `_`; its value +// the instance and the sequence of the state it was taken from. False when the state holds none of those fields. +func IndexEntryOf(on []string, key string, rec Record) (string, []byte, bool) { + var state map[string]any + if json.Unmarshal(rec.State, &state) != nil || state == nil { + return "", nil, false + } + parts := make([]string, 0, len(on)+1) + found := false + for _, f := range on { + v, has := state[f] + if !has { + parts = append(parts, "_") + continue + } + found = true + s, ok := v.(string) + if !ok { + raw, _ := json.Marshal(v) + s = string(raw) + } + if s = unsafe.ReplaceAllString(s, "_"); s == "" { + s = "_" + } + parts = append(parts, s) + } + if !found { + return "", nil, false + } + value, _ := json.Marshal(map[string]any{"instance": InstanceOf(key), "sequence": rec.Sequence}) + return strings.Join(append(parts, key), "."), value, true +} + +// declared is the fields an index is on and the types declaring it. +func (l *Ledger) declared(index string) ([]string, map[string]bool, error) { + types := map[string]bool{} + var on []string + for _, t := range l.Types { + for _, ix := range t.Indexes { + if ix.Name == index { + types[t.Name] = true + on = ix.On + } + } + } + if len(types) == 0 { + return nil, nil, fmt.Errorf("%s's ledger %s declares no index %q", l.Module, l.Name, index) + } + return on, types, nil +} + +// wanted is every entry an index should hold, from the states of the types declaring it. +func (l *Ledger) wanted(index string) (map[string][]byte, map[string]uint64, error) { + on, types, err := l.declared(index) + if err != nil { + return nil, nil, err + } + keys, err := l.instances() + if err != nil { + return nil, nil, err + } + want, seqs := map[string][]byte{}, map[string]uint64{} + for _, key := range keys { + name, _, _ := strings.Cut(key, ".") + if !types[name] { + continue + } + held, _, err := l.readState(key) + if err != nil { + return nil, nil, err + } + if k, v, ok := IndexEntryOf(on, key, held); ok { + want[k], seqs[k] = v, held.Sequence + } + } + return want, seqs, nil +} + +// Reindex builds a declared index again from the ledger's states, derived from its history: every entry it should hold +// written, every other removed. +func (l *Ledger) Reindex(r buscalls.ReindexRequest) (buscalls.Repaired, error) { + ix, err := l.index(r.Index) + if err != nil { + return buscalls.Repaired{}, err + } + want, _, err := l.wanted(r.Index) + if err != nil { + return buscalls.Repaired{}, err + } + held, err := ix.Entries("", 0) + if err != nil { + return buscalls.Repaired{}, err + } + var out buscalls.Repaired + for _, e := range held { + if _, keep := want[e.Key]; !keep { + if err := ix.Delete(e.Key); err != nil { + return out, err + } + } + } + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if err := ix.Put(k, want[k]); err != nil { + return out, err + } + out.Written++ + } + return out, nil +} + +// Lag is every instance whose state or snapshot is behind its history, and how many entries each declared index lacks +// or holds from an older state; nothing when nothing is behind. +func (l *Ledger) Lag(buscalls.LedgerRequest) (buscalls.Lagged, error) { + keys, err := l.instances() + if err != nil { + return buscalls.Lagged{}, err + } + var out buscalls.Lagged + for _, key := range keys { + last, err := l.History.Last(key) + if err != nil { + return out, err + } + held, _, err := l.readState(key) + if err != nil { + return out, err + } + snap, _, err := l.readSnapshot(key) + if err != nil { + return out, err + } + if held.Sequence < last || snap.Sequence < last { + out.Instances = append(out.Instances, buscalls.InstanceLag{Instance: InstanceOf(key), History: last, + State: held.Sequence, Snapshot: snap.Sequence}) + } + } + names := make([]string, 0, len(l.Indexes)) + for name := range l.Indexes { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, _, err := l.declared(name); err != nil { + // A bucket bound for an index no type declares has nothing it should hold. + continue + } + want, seqs, err := l.wanted(name) + if err != nil { + return out, err + } + held, err := l.Indexes[name].Entries("", 0) + if err != nil { + return out, err + } + at := map[string]uint64{} + for _, e := range held { + var v struct { + Sequence uint64 `json:"sequence"` + } + _ = json.Unmarshal(e.Value, &v) + at[e.Key] = v.Sequence + } + behind := uint64(0) + for k := range want { + if s, has := at[k]; !has || s < seqs[k] { + behind++ + } + } + if behind > 0 { + out.Indexes = append(out.Indexes, buscalls.IndexLag{Index: name, Behind: behind}) + } + } + return out, nil +} diff --git a/node-tools/internal/runtime/ledgers.go b/node-tools/internal/runtime/ledgers.go new file mode 100644 index 0000000..3b5b053 --- /dev/null +++ b/node-tools/internal/runtime/ledgers.go @@ -0,0 +1,146 @@ +package runtime + +import ( + "encoding/json" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "github.com/novox/mesh-tools/node-tools/internal/bus" + "github.com/novox/mesh-tools/node-tools/internal/launch" + "github.com/novox/mesh-tools/node-tools/internal/ledger" +) + +// A module's ledgers as the tool runner serves them (novox/hq ADR 0597, design 25 § 12.4, § 12.5; issue 1966): opened +// for the module whose bundle or whose repair tool asks, from its own bindings alone, with its types and the connection +// its history is published on. + +// LedgerTypes is where the runner reads a ledger's types, by module and local name. Where it reads them from on the +// live mesh is not decided (issue 1966: the binding carries no types, and no record names a source); until it is, +// the runner refuses to open a ledger and says why, and a test gives the types. +type LedgerTypes func(module, local string) ([]catalogue.LedgerType, error) + +// errTypesUndecided is the refusal of a ledger whose types the runner has no source for. +func errTypesUndecided(module, local string) error { + return fmt.Errorf("the tool runner cannot open %s's ledger %q: it reads a ledger's types from nowhere yet, since its "+ + "binding carries none and no record names their source (issue 1966)", module, local) +} + +// ledgerSource is how the runner opens a module's ledger: its types, and the connection its history and snapshots are +// published on (the module's ledger credential, ADR 0597 §9). +type ledgerSource struct { + types LedgerTypes + writer func(module string) (*bus.Conn, error) +} + +// ledger opens one of a module's ledgers by its local name. +func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) { + b, err := c.conn.Binding(module, local, catalogue.StorageLedger) + if err != nil { + return nil, err + } + types := c.ledgers.types + if types == nil { + return nil, errTypesUndecided(module, local) + } + ts, err := types(module, local) + if err != nil { + return nil, err + } + var writer *bus.Conn + if c.ledgers.writer != nil { + if writer, err = c.ledgers.writer(module); err != nil { + return nil, err + } + } + history, state, snapshots := c.conn.LedgerStores(b, writer) + return &ledger.Ledger{Module: module, Name: local, Types: ts, History: history, State: state, Snapshots: snapshots, + Indexes: c.conn.LedgerIndexes(b), Logf: c.logf}, nil +} + +// hasLedger is whether a module's membership binds a ledger. +func hasLedger(conn *bus.Conn, module string) bool { + for _, b := range conn.Bindings(module) { + if b.Storage == catalogue.StorageLedger { + return true + } + } + return false +} + +// The repair tools the runner serves for every module owning a ledger (design 25 § 12.5), named so no bundle's tool is +// taken for one. +const ( + RepairRebuild = "ledger_rebuild" + RepairReindex = "ledger_reindex" + RepairSnapshot = "ledger_snapshot" + RepairLag = "ledger_lag" +) + +// The effects a tool carries (novox/hq ADR 0387 §1). +const ( + EffectReads = "reads" + EffectActs = "acts" +) + +// repairTools are a module's ledger repairs as tools, each limited to its own ledgers: the request's ledger is a local +// name of the module whose tool is called. +func (c *consumers) repairTools(module string) []launch.Tool { + schema := func(props string, required ...string) json.RawMessage { + req, _ := json.Marshal(append([]string{"ledger"}, required...)) + return json.RawMessage(`{"type":"object","properties":{"ledger":{"type":"string","description":"the ledger's local name"}` + + props + `},"required":` + string(req) + `}`) + } + instance := `,"instance":{"type":"string","description":"one instance, such as decision/0290; every instance when empty"}` + run := func(do func(l *ledger.Ledger, args json.RawMessage) (any, error)) func(string, json.RawMessage) (json.RawMessage, error) { + return func(_ string, args json.RawMessage) (json.RawMessage, error) { + var named struct { + Ledger string `json:"ledger"` + } + if err := json.Unmarshal(args, &named); err != nil || named.Ledger == "" { + return nil, errors.New("a repair names the ledger, by its local name") + } + l, err := c.ledger(module, named.Ledger) + if err != nil { + return nil, err + } + answer, err := do(l, args) + if err != nil { + return nil, err + } + return json.Marshal(answer) + } + } + return []launch.Tool{ + {Name: RepairRebuild, Effect: EffectActs, Input: schema(instance + `,"from":{"type":"string","enum":["history","snapshot"],"description":"derive from the whole history (the default) or from the snapshot and the operations after it"}`), + Description: "Derive a ledger instance's state again from its history or its snapshot, and write it and its snapshot (novox/hq ADR 0597 §10).", + Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) { + var r buscalls.RebuildRequest + _ = json.Unmarshal(args, &r) + return l.Rebuild(r) + })}, + {Name: RepairReindex, Effect: EffectActs, Input: schema(`,"index":{"type":"string","description":"the declared index"}`, "index"), + Description: "Build a ledger's declared index again from its states, derived from its history (novox/hq ADR 0597 §4, §10).", + Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) { + var r buscalls.ReindexRequest + _ = json.Unmarshal(args, &r) + return l.Reindex(r) + })}, + {Name: RepairSnapshot, Effect: EffectActs, Input: schema(instance), + Description: "Rewrite the snapshot of a ledger instance that is behind its state (novox/hq ADR 0597 §8, §10).", + Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) { + var r buscalls.InstanceRequest + _ = json.Unmarshal(args, &r) + return l.Snapshot(r) + })}, + {Name: RepairLag, Effect: EffectReads, Input: schema(""), + Description: "Say every ledger instance whose state or snapshot is behind its history, and each declared index behind it (novox/hq ADR 0597 §10).", + Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) { + var r buscalls.LedgerRequest + _ = json.Unmarshal(args, &r) + return l.Lag(r) + })}, + } +} diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go new file mode 100644 index 0000000..15ba59f --- /dev/null +++ b/node-tools/internal/runtime/ledgers_test.go @@ -0,0 +1,74 @@ +package runtime + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" + + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// 1966/AC16: given a module with a ledger binding, when the runner lists its tools, then the four repair tools are +// among them, marked acts or reads; a module without one has none, and a binding written later brings them. +func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { + mesh := mt.New(t) + owner := mt.MembershipOf("owner", "anchor", true, nil) + owner.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_owner_records", Subject: "mesh.ledger.owner.records", State: "owner_records", + Snapshot: "mesh.ledger-snapshot.owner.records"}} + plain := mt.MembershipOf("plain", "anchor", true, nil) + mesh.Issue(t, owner) + mesh.Issue(t, plain) + nodeTools := connect(t, "node-tools", "anchor") + stop, err := Run(nodeTools, []Served{{"owner", []string{mt.Fixture("caller-echo.py")}}, + {"plain", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {}) + if err != nil { + t.Fatal(err) + } + defer stop() + asker := connect(t, "console", "workstation") + listed := func(module string) (string, error) { + answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "") + if err != nil { + return "", err + } + var got ToolsAnswer + if err := json.Unmarshal(answered.Result, &got); err != nil { + return "", err + } + var names []string + for _, tool := range got.Tools { + names = append(names, tool.Name+":"+tool.Effect) + } + sort.Strings(names) + return strings.Join(names, " "), nil + } + for _, c := range []struct{ name, module, want string }{ + {"a ledger owner lists the four repairs", "owner", "ledger_lag:reads ledger_rebuild:acts ledger_reindex:acts ledger_snapshot:acts who:"}, + {"a module with no ledger has none", "plain", "who:"}, + } { + t.Run(c.name, func(t *testing.T) { + var got string + mt.Until(t, func() (err error) { got, err = listed(c.module); return }) + if got != c.want { + t.Fatalf("%s lists %q, want %q", c.module, got, c.want) + } + }) + } + t.Run("a ledger bound later brings them", func(t *testing.T) { + plain.Bindings = owner.Bindings + mesh.Issue(t, plain) + mt.Until(t, func() error { + got, err := listed("plain") + if err == nil && !strings.Contains(got, "ledger_rebuild:acts") { + err = errors.New("listed " + got) + } + return err + }) + }) +} diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index de7f3a4..8df61b5 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -52,6 +52,8 @@ type ListedTool struct { Description string `json:"description"` Input json.RawMessage `json:"input"` Subjects []string `json:"subjects,omitempty"` + // Effect is the tool's mark where the runtime knows it (novox/hq ADR 0387 §1): its own tools, the ledger repairs. + Effect string `json:"effect,omitempty"` } // NodeRuntimeModule is the module that is a machine's runtime, serving every module on it. @@ -133,6 +135,60 @@ type registration struct { module string // the module's own name, or a seat's owner string // the module whose bundle made it tools []launch.Tool + // repairs says it is the runtime's own ledger repairs for the module, not a bundle's (ledgers.go). + repairs bool +} + +// withRepairs is regs with the module's ledger repairs as its binding now says: added when it binds a ledger, removed +// when it binds none. +func withRepairs(regs []registration, repairs *registration) []registration { + out := make([]registration, 0, len(regs)+1) + for _, r := range regs { + if !r.repairs { + out = append(out, r) + } + } + if repairs != nil { + out = append(out, *repairs) + } + return out +} + +// repairsOf is a module's ledger repairs as a registration, nil when its membership binds no ledger. +func (r *runner) repairsOf(module string) *registration { + if !hasLedger(r.conn, module) { + return nil + } + return ®istration{module: module, owner: module, tools: r.events.repairTools(module), repairs: true} +} + +// refreshRepairs serves each module's ledger repairs as its membership binds a ledger now, re-serving its tools where +// that changed (a binding is written when the bus module created the ledger, design 25 § 12.2). +func (r *runner) refreshRepairs() { + r.mu.Lock() + defer r.mu.Unlock() + changed := false + for _, m := range r.modules { + l := r.of[m] + served := false + for _, reg := range l.own { + served = served || reg.repairs + } + repairs := r.repairsOf(m) + if served == (repairs != nil) { + continue + } + for i := len(l.serving) - 1; i >= 0; i-- { + l.serving[i]() + } + l.serving, l.names = nil, nil + l.own, l.regs = withRepairs(l.own, repairs), withRepairs(l.regs, repairs) + r.serve(l) + changed = true + } + if changed { + r.bindVerbs() + } } // loadedModule is one served module as its bundles were last loaded: what they registered, what of it @@ -230,6 +286,7 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log line += fmt.Sprintf("; not serving %s, whose bundle(s) failed to load", strings.Join(failedNames, ", ")) } logf("%s", line) + conn.OnMembership(func(bus.Membership) { go func() { r.refreshRepairs(); conn.Flush() }() }) stopSeats, again := serveSeats(conn, r.modules, r.registrations, logf) r.seatsAgain = again r.stops = append(r.stops, stopSeats) @@ -341,6 +398,9 @@ func (r *runner) load(s Served) *loadedModule { } } } + if repairs := r.repairsOf(s.Module); repairs != nil { + l.regs, l.own = append(l.regs, *repairs), append(l.own, *repairs) + } return l } @@ -411,7 +471,7 @@ func (r *runner) bindVerbs() { answer := ToolsAnswer{Module: module, Tools: []ListedTool{}, Failed: why} for _, t := range tools { answer.Tools = append(answer.Tools, ListedTool{Name: t.Name, Description: t.Description, - Input: t.Input, Subjects: subjectsOf(r.conn, module, t.Name)}) + Input: t.Input, Subjects: subjectsOf(r.conn, module, t.Name), Effect: t.Effect}) } return answer, nil }) @@ -682,6 +742,8 @@ type consumers struct { seatBound map[string]func() // provisioned is every watch a module's code holds of its values at runtime (provision.go). provisioned provisionWatches + // ledgers is how a module's ledger is opened (ledgers.go). + ledgers ledgerSource } type moduleEvents struct { -- 2.54.0 From 38a634723b9a9e6da4acf61fb090129b230d30c5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:49:15 +0100 Subject: [PATCH 05/12] Stamp a ledger operation with the caller the runner proved for the call in flight it names, refusing another module's call or one not in flight; serve the ledger calls on the pipe (issue 1966 T5) --- .../internal/launch/event_answer_test.go | 1 + node-tools/internal/launch/launch.go | 24 ++- node-tools/internal/ledger/ledger.go | 6 +- node-tools/internal/runtime/busstate.go | 5 + node-tools/internal/runtime/caller_test.go | 7 + node-tools/internal/runtime/ledgers.go | 148 +++++++++++++++++- node-tools/internal/runtime/ledgers_test.go | 103 ++++++++++++ node-tools/internal/runtime/runtime.go | 2 + node-tools/test/fixtures/caller-echo.py | 5 +- 9 files changed, 290 insertions(+), 11 deletions(-) diff --git a/node-tools/internal/launch/event_answer_test.go b/node-tools/internal/launch/event_answer_test.go index 6746b36..0824263 100644 --- a/node-tools/internal/launch/event_answer_test.go +++ b/node-tools/internal/launch/event_answer_test.go @@ -52,6 +52,7 @@ func (b *subscribing) BusState(string, json.RawMessage) (json.RawMessage, error) func (b *subscribing) BusStateWatch(string, json.RawMessage, func(json.RawMessage) error) (func(), error) { return func() {}, nil } +func (b *subscribing) Begin(string) (string, func()) { return "", func() {} } func (b *subscribing) Subscribe(d func(json.RawMessage) error) error { b.mu.Lock() defer b.mu.Unlock() diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 17d95d4..f663c82 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -52,12 +52,21 @@ type Tool struct { // stdio.CallerOf, and says Unproven when it is absent. const MetaCaller = "mesh/caller" -// callParams is a `tools/call`'s params: the tool, its arguments as the caller sent them, and — only when the -// subject named one — the caller, in `_meta`, which the runtime writes and no caller can. -func callParams(name, caller string, args json.RawMessage) map[string]any { +// callParams is a `tools/call`'s params: the tool, its arguments as the caller sent them, and in `_meta`, which the +// runtime writes and no caller can, the caller — only when the subject named one — and the id of the call in flight +// (mesh-sdk buscalls.MetaCall), which a ledger's apply names so the runtime stamps the caller it proved (novox/hq ADR +// 0597 §7). +func callParams(name, caller, call string, args json.RawMessage) map[string]any { params := map[string]any{"name": name, "arguments": args} + meta := map[string]any{} if caller != "" { - params["_meta"] = map[string]any{MetaCaller: caller} + meta[MetaCaller] = caller + } + if call != "" { + meta[buscalls.MetaCall] = call + } + if len(meta) > 0 { + params["_meta"] = meta } return params } @@ -101,6 +110,9 @@ type Bus interface { BusState(method string, params json.RawMessage) (json.RawMessage, error) // BusStateWatch hands deliver each change of an objects or a ledger watch, the current ones first. BusStateWatch(method string, params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error) + // Begin says a tool call of the module is in flight for the caller the runtime proved, and answers its id, which the + // bundle is handed in the call's `_meta`, and the end to call once the call is answered (novox/hq ADR 0597 §7). + Begin(caller string) (id string, end func()) } // busStateCalls are the methods of mesh-sdk buscalls a bundle asks and BusState answers. @@ -659,7 +671,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an if len(args) == 0 || string(args) == "null" { args = json.RawMessage("{}") } - res, err := asking("tools/call", callParams(full, caller, args), CallTimeout) + call, end := mesh.Begin(caller) + defer end() + res, err := asking("tools/call", callParams(full, caller, call, args), CallTimeout) if err != nil { return nil, err } diff --git a/node-tools/internal/ledger/ledger.go b/node-tools/internal/ledger/ledger.go index f39b889..3088a6f 100644 --- a/node-tools/internal/ledger/ledger.go +++ b/node-tools/internal/ledger/ledger.go @@ -81,8 +81,10 @@ type Snapshots interface { // Ledger is one ledger of one module: its types and where it is kept. type Ledger struct { - Module string - Name string + Module string + Name string + // Account is the owning module's authenticated account, which every operation it applies is stamped with. + Account string Types []catalogue.LedgerType History History State State diff --git a/node-tools/internal/runtime/busstate.go b/node-tools/internal/runtime/busstate.go index 905d3f0..bd72a15 100644 --- a/node-tools/internal/runtime/busstate.go +++ b/node-tools/internal/runtime/busstate.go @@ -65,6 +65,9 @@ func (b *moduleBus) BusState(method string, params json.RawMessage) (json.RawMes if err = decode(method, params, &r); err == nil { answer, err = conn.WorkQueueDead(b.module, r) } + case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerHistory, buscalls.LedgerQuery, buscalls.LedgerRebuild, + buscalls.LedgerReindex, buscalls.LedgerSnapshot, buscalls.LedgerLag: + answer, err = b.ledgerCall(method, params) default: return nil, fmt.Errorf("the runtime answers no %s", method) } @@ -89,6 +92,8 @@ func (b *moduleBus) BusStateWatch(method string, params json.RawMessage, deliver } return deliver(raw) }) + case buscalls.LedgerWatch: + return b.ledgerWatch(params, deliver) } return nil, fmt.Errorf("the runtime answers no %s", method) } diff --git a/node-tools/internal/runtime/caller_test.go b/node-tools/internal/runtime/caller_test.go index 8228231..7958b39 100644 --- a/node-tools/internal/runtime/caller_test.go +++ b/node-tools/internal/runtime/caller_test.go @@ -13,6 +13,7 @@ import ( type saidCaller struct { Caller string `json:"caller"` + Call string `json:"call"` Arguments map[string]any `json:"arguments"` } @@ -36,6 +37,7 @@ func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T) asker.Logf = func(string, ...any) {} defer asker.Close() + calls := map[string]bool{} for _, key := range []string{"ledger.who", "ledger.who@anchor", "seat:node-desk.who@anchor"} { var got saidCaller mt.Until(t, func() error { @@ -49,6 +51,11 @@ func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T) if got.Caller != "shanks.node-tools" { t.Errorf("%s reached the module with caller %q, want the asker's bus user", key, got.Caller) } + // And the id of the call in flight, one per call, which a ledger's apply names (novox/hq ADR 0597 §7). + if got.Call == "" || calls[got.Call] { + t.Errorf("%s reached the module with call id %q, want one of its own", key, got.Call) + } + calls[got.Call] = true if got.Arguments["caller"] != "novox.node-tools" { t.Errorf("%s: the arguments did not reach the module as sent: %v", key, got.Arguments) } diff --git a/node-tools/internal/runtime/ledgers.go b/node-tools/internal/runtime/ledgers.go index 3b5b053..7c15a1b 100644 --- a/node-tools/internal/runtime/ledgers.go +++ b/node-tools/internal/runtime/ledgers.go @@ -1,9 +1,12 @@ package runtime import ( + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" + "sync" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -35,6 +38,59 @@ type ledgerSource struct { writer func(module string) (*bus.Conn, error) } +// callsInFlight are the tool calls the runtime is serving, each with the module whose bundle serves it and the caller +// the runtime proved from the subject the call arrived on (bus.CalledSubject, novox/hq issue 365). +type callsInFlight struct { + mu sync.Mutex + of map[string]callInFlight +} + +type callInFlight struct{ module, caller string } + +// begin keeps a call of module in flight for caller, under an id no bundle can guess, until end. +func (f *callsInFlight) begin(module, caller string) (string, func()) { + var raw [16]byte + _, _ = rand.Read(raw[:]) + id := hex.EncodeToString(raw[:]) + f.mu.Lock() + if f.of == nil { + f.of = map[string]callInFlight{} + } + f.of[id] = callInFlight{module, caller} + f.mu.Unlock() + return id, func() { + f.mu.Lock() + delete(f.of, id) + f.mu.Unlock() + } +} + +// authorOf is the author an apply of module naming call is stamped with (ADR 0597 §7): the module's account, and the +// caller the runtime proved for that call; naming no call, the account alone. A call not in flight, or another +// module's, is refused, naming it. Nothing the bundle writes is taken as the author. +func (f *callsInFlight) authorOf(module, account, call string) (buscalls.Author, error) { + author := buscalls.Author{Account: account} + if call == "" { + return author, nil + } + f.mu.Lock() + inFlight, ok := f.of[call] + f.mu.Unlock() + switch { + case !ok: + return author, fmt.Errorf("the apply names the call %s, which is not a tool call in flight here; an apply names the "+ + "call it serves as its tools/call's _meta %s gave it, or none (novox/hq ADR 0597 §7)", call, buscalls.MetaCall) + case inFlight.module != module: + return author, fmt.Errorf("the apply names the call %s, which is another module's tool call; %s names only its own "+ + "(novox/hq ADR 0597 §7)", call, module) + } + author.Caller = inFlight.caller + return author, nil +} + +// Begin keeps one of the module's tool calls in flight for the caller the runtime proved (launch.Bus). +func (b *moduleBus) Begin(caller string) (string, func()) { return b.all.calls.begin(b.module, caller) } + // ledger opens one of a module's ledgers by its local name. func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) { b, err := c.conn.Binding(module, local, catalogue.StorageLedger) @@ -56,8 +112,96 @@ func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) { } } history, state, snapshots := c.conn.LedgerStores(b, writer) - return &ledger.Ledger{Module: module, Name: local, Types: ts, History: history, State: state, Snapshots: snapshots, - Indexes: c.conn.LedgerIndexes(b), Logf: c.logf}, nil + // The owning module's authenticated account: the user of the connection its history is published on. + account := module + if writer != nil && writer.Credential().User != "" { + account = writer.Credential().User + } + return &ledger.Ledger{Module: module, Name: local, Account: account, Types: ts, History: history, State: state, + Snapshots: snapshots, Indexes: c.conn.LedgerIndexes(b), Logf: c.logf}, nil +} + +// ledgerCall answers a bundle's ledger call, as the module whose bundle asks. +func (b *moduleBus) ledgerCall(method string, params json.RawMessage) (any, error) { + var named struct { + Ledger string `json:"ledger"` + } + if err := decode(method, params, &named); err != nil { + return nil, err + } + if method == buscalls.LedgerApply { + var r buscalls.ApplyRequest + if err := decode(method, params, &r); err != nil { + return nil, err + } + // The call is judged first: a forged one is refused before anything of the ledger is touched. + if _, err := b.all.calls.authorOf(b.module, "", r.Call); err != nil { + return nil, err + } + l, err := b.all.ledger(b.module, r.Ledger) + if err != nil { + return nil, err + } + author, err := b.all.calls.authorOf(b.module, l.Account, r.Call) + if err != nil { + return nil, err + } + return l.Apply(r, author) + } + l, err := b.all.ledger(b.module, named.Ledger) + if err != nil { + return nil, err + } + switch method { + case buscalls.LedgerGet: + var r buscalls.InstanceRequest + _ = json.Unmarshal(params, &r) + return l.Get(r) + case buscalls.LedgerHistory: + var r buscalls.HistoryRequest + _ = json.Unmarshal(params, &r) + return l.ReadHistory(r) + case buscalls.LedgerQuery: + var r buscalls.QueryRequest + _ = json.Unmarshal(params, &r) + return l.Query(r) + case buscalls.LedgerRebuild: + var r buscalls.RebuildRequest + _ = json.Unmarshal(params, &r) + return l.Rebuild(r) + case buscalls.LedgerReindex: + var r buscalls.ReindexRequest + _ = json.Unmarshal(params, &r) + return l.Reindex(r) + case buscalls.LedgerSnapshot: + var r buscalls.InstanceRequest + _ = json.Unmarshal(params, &r) + return l.Snapshot(r) + case buscalls.LedgerLag: + var r buscalls.LedgerRequest + _ = json.Unmarshal(params, &r) + return l.Lag(r) + } + return nil, fmt.Errorf("the runtime answers no %s", method) +} + +// ledgerWatch hands a bundle each change of its ledger's states. +func (b *moduleBus) ledgerWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) { + var r buscalls.LedgerWatchRequest + if err := decode(buscalls.LedgerWatch, params, &r); err != nil { + return nil, err + } + l, err := b.all.ledger(b.module, r.Ledger) + if err != nil { + return nil, err + } + return l.Watch(r, func(change buscalls.InstanceChange) error { + raw, err := json.Marshal(change) + if err != nil { + return err + } + return deliver(raw) + }) } // hasLedger is whether a module's membership binds a ledger. diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index 15ba59f..789cece 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -1,15 +1,20 @@ package runtime import ( + "context" "encoding/json" "errors" "sort" "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-tools/node-tools/internal/bus" mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" ) @@ -72,3 +77,101 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { }) }) } + +// aLedger raises one ledger of module m on the bus as the bus module would create it, binds it in m's membership, and +// answers the runtime's buses for m and for other, its types given as a test gives them. +func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { + t.Helper() + mesh = mt.New(t) + nc, err := nats.Connect(mt.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, _ := jetstream.New(nc) + ctx := context.Background() + for _, cfg := range []jetstream.StreamConfig{ + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + } { + if _, err := js.CreateStream(ctx, cfg); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records"}); err != nil { + t.Fatal(err) + } + mem := mt.MembershipOf("m", "anchor", false, nil) + mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} + mesh.Issue(t, mem) + mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil)) + conn := connect(t, "node-tools", "anchor") + conn.Follow("m") + conn.Follow("other") + mt.Until(t, func() error { + if conn.Membership("m") == nil || conn.Membership("other") == nil { + return errors.New("not issued yet") + } + return nil + }) + all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}, ledgers: ledgerSource{ + types: func(string, string) ([]catalogue.LedgerType, error) { + return []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, + Operations: []catalogue.LedgerOperation{{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}}}}, nil + }, + writer: func(string) (*bus.Conn, error) { return conn, nil }, + }} + return all.forModule("m").(*moduleBus), all.forModule("other").(*moduleBus), mesh +} + +// 1966/AC17, 1966/AC18: the author an apply is stamped with is the caller the runtime proved for the call in flight it +// names, whatever author the bundle wrote; another module's call, or one not in flight, is refused naming it (ADR 0597 §7). +func TestAnApplyIsStampedWithTheCallerTheRuntimeProved(t *testing.T) { + m, other, _ := aLedger(t) + call, end := m.Begin("op@shanks") + defer end() + theirs, endTheirs := other.Begin("op@shanks") + defer endTheirs() + forged := `"body":{"title":"T","author":{"account":"x","caller":"forged"}}` + for _, c := range []struct{ name, params, caller, refusal string }{ + {"1966/AC17 a call in flight stamps its proven caller", `{"ledger":"records","instance":"decision/1","operation":"title",` + forged + `,"call":"` + call + `","author":"forged"}`, "op@shanks", ""}, + {"naming no call stamps the account alone", `{"ledger":"records","instance":"decision/2","operation":"title",` + forged + `}`, "", ""}, + {"1966/AC18 another module's call is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"` + theirs + `"}`, "", "the call " + theirs + ", which is another module's"}, + {"a call not in flight is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"c0ffee"}`, "", "the call c0ffee, which is not a tool call in flight"}, + } { + t.Run(c.name, func(t *testing.T) { + var req struct { + Instance string `json:"instance"` + } + _ = json.Unmarshal([]byte(c.params), &req) + _, err := m.BusState(buscalls.LedgerApply, json.RawMessage(c.params)) + if c.refusal != "" { + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Fatalf("applied with %v, want refused with %q", err, c.refusal) + } + return + } + if err != nil { + t.Fatal(err) + } + raw, err := m.BusState(buscalls.LedgerHistory, json.RawMessage(`{"ledger":"records","instance":"`+req.Instance+`"}`)) + if err != nil { + t.Fatal(err) + } + var page buscalls.HistoryPage + _ = json.Unmarshal(raw, &page) + if len(page.Operations) != 1 || page.Operations[0].Author != (buscalls.Author{Account: "m", Caller: c.caller}) { + t.Fatalf("the history holds %+v, want the author {m %s}", page.Operations, c.caller) + } + state, _ := m.BusState(buscalls.LedgerGet, json.RawMessage(`{"ledger":"records","instance":"`+req.Instance+`"}`)) + if !strings.Contains(string(state), `"author":{"account":"m"`) || strings.Contains(string(state), "forged") { + t.Fatalf("the state took the bundle's author: %s", state) + } + }) + } + end() + if _, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"decision/4","operation":"title","call":"`+call+`"}`)); err == nil { + t.Fatal("a call that ended was still taken as in flight") + } +} diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index 8df61b5..b7d6e41 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -744,6 +744,8 @@ type consumers struct { provisioned provisionWatches // ledgers is how a module's ledger is opened (ledgers.go). ledgers ledgerSource + // calls are the tool calls in flight, by the id each bundle was handed (ledgers.go, novox/hq ADR 0597 §7). + calls callsInFlight } type moduleEvents struct { diff --git a/node-tools/test/fixtures/caller-echo.py b/node-tools/test/fixtures/caller-echo.py index 16706cd..a7f7c3e 100755 --- a/node-tools/test/fixtures/caller-echo.py +++ b/node-tools/test/fixtures/caller-echo.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # A tools bundle that says who the runtime said called it (novox/hq issue 365): MCP over stdio, no SDK. Its tool -# and its seat verb answer the caller the call's `_meta` named — "" when it named none — and the arguments as given. +# and its seat verb answer the caller the call's `_meta` named — "" when it named none —, the id of the call in flight +# it named (novox/hq ADR 0597 §7), and the arguments as given. import json, sys def say(m): @@ -18,7 +19,7 @@ for line in sys.stdin: say({"id": rid, "result": {"tools": TOOLS}}) elif m == "tools/call": meta = p.get("_meta") or {} - said = {"caller": meta.get("mesh/caller", ""), "arguments": p.get("arguments") or {}} + said = {"caller": meta.get("mesh/caller", ""), "call": meta.get("mesh/call", ""), "arguments": p.get("arguments") or {}} say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps(said)}]}}) elif rid is not None: say({"id": rid, "result": {}}) -- 2.54.0 From 2bb0a6461df9254078877bba048708a669f9e560 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:51:47 +0100 Subject: [PATCH 06/12] Publish a module's ledger history and snapshots only on its own ledger connection, read from the runner's own secret ledgers and followed without a restart (issue 1966 T6) --- node-tools/cmd/node-tools/main.go | 11 +- node-tools/internal/bus/bus.go | 16 +- .../internal/runtime/ledger_credentials.go | 143 ++++++++++++++++++ .../runtime/ledger_credentials_test.go | 137 +++++++++++++++++ node-tools/internal/runtime/runtime.go | 20 ++- node-tools/module.json | 3 +- 6 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 node-tools/internal/runtime/ledger_credentials.go create mode 100644 node-tools/internal/runtime/ledger_credentials_test.go diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index f053f93..bf55106 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -5,6 +5,7 @@ // wherever MESH_CONSOLE_LISTEN says — it is also the console, MCP over HTTP on loopback. // // MESH_BROKER_FILE the credential the mesh sealed to this machine for the runtime +// MESH_LEDGERS_FILE its own secret ledgers, each module's ledger credential; beside the broker file when unset // MESH_TOOL_MODULES =,… the bundles to launch // MESH_TOOL_ENV {"": {"": ""}} what each is given (ADR 0192) // MESH_OPERATOR_ACCOUNT whose machine this is, and MESH_OPERATOR_HOME their home @@ -56,7 +57,15 @@ func main() { if err != nil { log.Fatalf("node-tools: %v", err) } - stop, err := runtime.Run(conn, served, envs, log.Printf) + // Its own secret ledgers: each module's ledger connection, the only one its history is published on (novox/hq ADR + // 0597 §9). Followed, so a credential minted or retired is taken with no restart. + var opts []runtime.Option + if file := runtime.LedgersFileOf(os.Getenv("MESH_BROKER_FILE")); file != "" { + ledgers := runtime.FollowLedgerCredentials(file, bus.Dial, log.Printf) + defer ledgers.Stop() + opts = append(opts, runtime.WithLedgerWriters(ledgers.Writer)) + } + stop, err := runtime.Run(conn, served, envs, log.Printf, opts...) if err != nil { log.Fatalf("node-tools: %v", err) } diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 8027e53..d5e5e3d 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -288,9 +288,20 @@ func (c *Conn) refused(err error) { }) } -// Connect dials the bus as the credential's module. A module's subjects come from its credential, -// never from its calls (ADR 0074). +// Connect dials the bus as the credential's module and follows its membership. A module's subjects come from its +// credential, never from its calls (ADR 0074). func Connect(cred Credential) (*Conn, error) { + c, err := Dial(cred) + if err != nil { + return nil, err + } + c.Follow(cred.Module) + return c, nil +} + +// Dial dials the bus as the credential's module and follows nothing: a connection that only publishes for its module, +// as a module's ledger connection does (novox/hq ADR 0597 §9). +func Dial(cred Credential) (*Conn, error) { if cred.Module == "" { return nil, errors.New("a broker credential with no module: the runtime derives its subjects " + "from the account the mesh issued, and cannot guess which module it is") @@ -329,7 +340,6 @@ func Connect(cred Credential) (*Conn, error) { } c = &Conn{nc: nc, js: js, self: cred.Module, node: cred.Node, cred: cred, issued: map[string]*Membership{}, Logf: log.Printf, answering: map[string]*answered{}} - c.Follow(cred.Module) return c, nil } diff --git a/node-tools/internal/runtime/ledger_credentials.go b/node-tools/internal/runtime/ledger_credentials.go new file mode 100644 index 0000000..3abca81 --- /dev/null +++ b/node-tools/internal/runtime/ledger_credentials.go @@ -0,0 +1,143 @@ +package runtime + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + + "github.com/novox/mesh-tools/node-tools/internal/bus" +) + +// The ledger credentials (novox/hq ADR 0597 §9, design 25 § 12.4; issue 1966 T6): one per module assignment, held by +// the machine's tool runner in its own secret `ledgers`, a map of module to credential (`{}` when none), which the vault +// rewrites on each mint and retirement. **A module's ledger history is appended, and its snapshots written, only on +// that module's own ledger connection**; everything else stays on the runner's own. + +// LedgersFile is where the runner reads its own secret `ledgers`: MESH_LEDGERS_FILE when set, else beside its broker +// credential, both in its directory mesh-state (node-tools' manifest: `${dir:mesh-state}/ledgers`). +const LedgersFile = "MESH_LEDGERS_FILE" + +// LedgersFileOf is the path of the secret `ledgers` for a runtime whose broker credential is at broker. +func LedgersFileOf(broker string) string { + if file := os.Getenv(LedgersFile); file != "" { + return file + } + if broker == "" { + return "" + } + return filepath.Join(filepath.Dir(broker), "ledgers") +} + +// LedgerCredentials are the ledger connections of the modules on this machine, read from the secret `ledgers` and +// followed as it changes, with no restart. +type LedgerCredentials struct { + file string + dial func(bus.Credential) (*bus.Conn, error) + logf func(string, ...any) + + mu sync.Mutex + creds map[string]bus.Credential + conns map[string]*bus.Conn + why string // why the file could not be read, when it could not + stop func() +} + +// FollowLedgerCredentials reads the secret `ledgers` at file and follows it (WatchFile), dialling a module's connection +// with dial the first time it is asked for. +func FollowLedgerCredentials(file string, dial func(bus.Credential) (*bus.Conn, error), logf func(string, ...any)) *LedgerCredentials { + l := &LedgerCredentials{file: file, dial: dial, logf: logf, creds: map[string]bus.Credential{}, conns: map[string]*bus.Conn{}} + l.read() + stop, err := WatchFile(file, l.read, logf, "a change of the ledger credentials is taken only after a restart") + if err != nil { + logf("[mesh-tools] cannot follow the ledger credentials at %s: %v; a change is taken only after a restart", file, err) + } else { + l.stop = stop + } + return l +} + +// read takes the file as it is now: a module whose credential changed or went has its connection closed, and is +// dialled again from the new one when next asked for. +func (l *LedgerCredentials) read() { + creds, why := map[string]bus.Credential{}, "" + raw, err := os.ReadFile(l.file) + switch { + case errors.Is(err, os.ErrNotExist): + why = "the file " + l.file + " is not there yet" + case err != nil: + why = err.Error() + case len(bytes.TrimSpace(raw)) == 0: + why = "the file " + l.file + " is empty" + default: + if err := json.Unmarshal(raw, &creds); err != nil { + why = "the file " + l.file + " is not a map of module to credential: " + err.Error() + creds = map[string]bus.Credential{} + } + } + l.mu.Lock() + var closing []*bus.Conn + for module, conn := range l.conns { + if now, has := creds[module]; !has || !reflect.DeepEqual(now, l.creds[module]) { + closing = append(closing, conn) + delete(l.conns, module) + } + } + saidBefore := l.why + l.creds, l.why = creds, why + l.mu.Unlock() + for _, c := range closing { + c.Close() + } + if why != "" && why != saidBefore { + l.logf("[mesh-tools] the ledger credentials cannot be read (%s); no ledger is written until they can", why) + } +} + +// Writer is a module's ledger connection, dialled from its credential the first time it is asked for; refused, naming +// what is missing, when the secret holds none for it. +func (l *LedgerCredentials) Writer(module string) (*bus.Conn, error) { + l.mu.Lock() + defer l.mu.Unlock() + if c := l.conns[module]; c != nil { + return c, nil + } + cred, has := l.creds[module] + if !has { + why := "" + if l.why != "" { + why = " (" + l.why + ")" + } + return nil, fmt.Errorf("%s's ledger credential is missing from the tool runner's own secret ledgers%s: its history "+ + "is published only on its own ledger connection, which the grants part issues with its membership and the "+ + "vault writes there (novox/hq ADR 0597 §9)", module, why) + } + if cred.Module == "" { + cred.Module = module + } + c, err := l.dial(cred) + if err != nil { + return nil, fmt.Errorf("%s's ledger connection could not be opened: %w", module, err) + } + c.Logf = l.logf + l.conns[module] = c + return c, nil +} + +// Stop ends the following and closes every ledger connection. +func (l *LedgerCredentials) Stop() { + if l.stop != nil { + l.stop() + } + l.mu.Lock() + conns := l.conns + l.conns = map[string]*bus.Conn{} + l.mu.Unlock() + for _, c := range conns { + c.Close() + } +} diff --git a/node-tools/internal/runtime/ledger_credentials_test.go b/node-tools/internal/runtime/ledger_credentials_test.go new file mode 100644 index 0000000..a517f7c --- /dev/null +++ b/node-tools/internal/runtime/ledger_credentials_test.go @@ -0,0 +1,137 @@ +package runtime + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-tools/node-tools/internal/bus" + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// 1966/AC22: given the secret ledgers holding m's credential, when m applies, then the append goes on m's connection +// (the runner's own account may not publish on m's ledger, as the grants part grants it); given it missing, apply is +// refused naming it; a change of the file is taken without a restart. +func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { + ledgerSubjects := []string{"mesh.ledger.m.records.>", "mesh.ledger-snapshot.m.records.>"} + users := []*server.User{ + {Username: "node-tools", Password: "pw", Permissions: &server.Permissions{ + Publish: &server.SubjectPermission{Allow: []string{">"}, Deny: ledgerSubjects}}}, + {Username: "ledger.anchor.m", Password: "pw", Permissions: &server.Permissions{ + Publish: &server.SubjectPermission{Allow: append([]string{"$JS.API.>", "_INBOX.>"}, ledgerSubjects...)}}}, + {Username: "admin", Password: "pw"}, + } + s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), + NoLog: true, NoSigs: true, Users: users}) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(30 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + url := s.ClientURL() + + admin, err := nats.Connect(url, nats.UserInfo("admin", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(admin.Close) + js, _ := jetstream.New(admin) + ctx := context.Background() + for _, cfg := range []jetstream.StreamConfig{ + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + } { + if _, err := js.CreateStream(ctx, cfg); err != nil { + t.Fatal(err) + } + } + for _, b := range []string{"m_records", buscompose.GrantsMembershipsBucket} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + mem := mt.MembershipOf("m", "anchor", false, nil) + mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} + raw, _ := json.Marshal(mem) + var sdk membership.Membership + _ = json.Unmarshal(raw, &sdk) + grants, _ := js.KeyValue(ctx, buscompose.GrantsMembershipsBucket) + if _, err := grants.Put(ctx, buscompose.MembershipKey("anchor", "m"), membership.EncodeGrantsMembership(sdk)); err != nil { + t.Fatal(err) + } + + conn, err := bus.Connect(bus.Credential{URL: url, Module: "node-tools", Node: "anchor", User: "node-tools", Password: "pw"}) + if err != nil { + t.Fatal(err) + } + conn.Logf = func(string, ...any) {} + t.Cleanup(conn.Close) + conn.Follow("m") + mt.Until(t, func() error { + if conn.Membership("m") == nil { + return os.ErrNotExist + } + return nil + }) + + file := filepath.Join(t.TempDir(), "ledgers") + if err := os.WriteFile(file, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + creds := FollowLedgerCredentials(file, bus.Dial, t.Logf) + t.Cleanup(creds.Stop) + types := func(string, string) ([]catalogue.LedgerType, error) { + return []catalogue.LedgerType{{Name: "decision", Operations: []catalogue.LedgerOperation{ + {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}}}}}, nil + } + all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}} + WithLedgerTypes(types)(all) + WithLedgerWriters(creds.Writer)(all) + m := all.forModule("m").(*moduleBus) + apply := func(instance string) error { + _, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"`+instance+`","operation":"title","body":{"title":"T"}}`)) + return err + } + + // Missing: refused, naming it. + if err := apply("decision/1"); err == nil || !strings.Contains(err.Error(), "m's ledger credential is missing from the tool runner's own secret ledgers") { + t.Fatalf("an apply with no credential: %v", err) + } + // Written: taken without a restart, on m's connection (the runner's own account is denied m's ledger subjects, so an + // append that lands went on m's), whose account stamps the operation. + cred, _ := json.Marshal(map[string]bus.Credential{"m": {URL: url, Node: "anchor", Module: "m", User: "ledger.anchor.m", Password: "pw"}}) + if err := os.WriteFile(file, cred, 0o600); err != nil { + t.Fatal(err) + } + mt.Until(t, func() error { return apply("decision/1") }) + page, err := m.BusState(buscalls.LedgerHistory, json.RawMessage(`{"ledger":"records","instance":"decision/1"}`)) + if err != nil || !strings.Contains(string(page), `"author":{"account":"ledger.anchor.m"}`) { + t.Fatalf("the history holds %s, %v", page, err) + } + // Gone again: refused again, the change taken without a restart. + if err := os.WriteFile(file, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + mt.Until(t, func() error { + if err := apply("decision/2"); err == nil || !strings.Contains(err.Error(), "credential is missing") { + return os.ErrExist + } + return nil + }) +} diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index b7d6e41..7e4756e 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -61,7 +61,7 @@ const NodeRuntimeModule = "node-tools" // runtimeOnly are the words of the runtime's own environment no bundle is given. var runtimeOnly = map[string]bool{ToolEnv: true, ToolModules: true, "MESH_BROKER_FILE": true, - "MESH_BROKER_URL": true, "MESH_CONSOLE_LISTEN": true} + "MESH_BROKER_URL": true, "MESH_CONSOLE_LISTEN": true, LedgersFile: true} // ServedModulesFrom reads MESH_TOOL_MODULES: `=` entries, comma-separated, several // per module. The one-module form — a bare path, or the runtime's own module — is the per-module @@ -234,9 +234,22 @@ type runner struct { stops []func() // the watch, the announcement and the seats, in the order they began } +// Option sets how Run serves, beside what it is given. +type Option func(*consumers) + +// WithLedgerWriters gives the runtime the connection each module's ledger history is published on (ledger_credentials.go). +func WithLedgerWriters(writer func(module string) (*bus.Conn, error)) Option { + return func(c *consumers) { c.ledgers.writer = writer } +} + +// WithLedgerTypes gives the runtime where it reads a ledger's types (ledgers.go). +func WithLedgerTypes(types LedgerTypes) Option { + return func(c *consumers) { c.ledgers.types = types } +} + // Run launches, binds and serves, and serves each module's bundle again when it appears, changes or // goes, with no restart. It answers a stop function. -func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any)) (func(), error) { +func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any), opts ...Option) (func(), error) { if account := os.Getenv(OperatorAccount); account != "" { home := "" if h := os.Getenv(OperatorHome); h != "" { @@ -259,6 +272,9 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log // The module's events, for every child of it that subscribes (ADR 0198): one consumer per module, // bound the first time any of its children subscribes, each event handed to every child that did. r.events = &consumers{conn: conn, logf: logf, of: map[string]*moduleEvents{}} + for _, opt := range opts { + opt(r.events) + } for _, s := range served { r.of[s.Module] = r.load(s) diff --git a/node-tools/module.json b/node-tools/module.json index e6cb6db..e456563 100644 --- a/node-tools/module.json +++ b/node-tools/module.json @@ -28,7 +28,8 @@ } }, "own-secrets": { - "broker": "${dir:mesh-state}/broker" + "broker": "${dir:mesh-state}/broker", + "ledgers": "${dir:mesh-state}/ledgers" }, "listens": [ { -- 2.54.0 From d206b0dde1e336a36f5477da63c53e0d011773de Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:52:58 +0100 Subject: [PATCH 07/12] Prove against the real bus that the ledger keeps its promises: one writer wins, the state never goes back, a cut or full state catches up, the snapshot rebuilds, the repairs repair, a forged author is refused (issue 1966 T7) --- .../runtime/ledger_integration_test.go | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 node-tools/internal/runtime/ledger_integration_test.go diff --git a/node-tools/internal/runtime/ledger_integration_test.go b/node-tools/internal/runtime/ledger_integration_test.go new file mode 100644 index 0000000..11d8153 --- /dev/null +++ b/node-tools/internal/runtime/ledger_integration_test.go @@ -0,0 +1,261 @@ +//go:build integration + +package runtime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-tools/node-tools/internal/bus" + "github.com/novox/mesh-tools/node-tools/internal/ledger" + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// racing is a history whose append lets another writer apply first, once: before its own append, or after it. +type racing struct { + ledger.History + before, after func() +} + +func (r *racing) Append(key string, op []byte, expect uint64) (uint64, error) { + if f := r.before; f != nil { + r.before = nil + f() + } + seq, err := r.History.Append(key, op, expect) + if f := r.after; f != nil && err == nil { + r.after = nil + f() + } + return seq, err +} + +// cut is a state whose next write is lost, as a writer stopped between its history and its state would lose it. +type cut struct { + ledger.State + once bool +} + +func (c *cut) Write(key string, value []byte, revision uint64) (uint64, error) { + if c.once { + c.once = false + return 0, errors.New("the writer stopped") + } + return c.State.Write(key, value, revision) +} + +// 1966/AC19: the ledger keeps its promises against the real bus (ADR 0597's How it is checked; design 25 § 12.7), one +// case each, and every apply leaves the snapshot equal to the state. +func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { + mesh := mt.New(t) + nc, err := nats.Connect(mt.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, _ := jetstream.New(nc) + ctx := context.Background() + for _, cfg := range []jetstream.StreamConfig{ + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + } { + if _, err := js.CreateStream(ctx, cfg); err != nil { + t.Fatal(err) + } + } + for _, b := range []string{"m_records", "m_records_by_title"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + binding := membership.BusBinding{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", + Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger-snapshot.m.records"} + mem := mt.MembershipOf("m", "anchor", false, nil) + mem.Bindings = []membership.BusBinding{binding} + mesh.Issue(t, mem) + conn := connect(t, "node-tools", "anchor") + conn.Follow("m") + mt.Until(t, func() error { + if conn.Membership("m") == nil { + return errors.New("not issued yet") + } + return nil + }) + types := []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, + Operations: []catalogue.LedgerOperation{ + {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}, + {Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}}, + }, + Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}} + all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}} + WithLedgerTypes(func(string, string) ([]catalogue.LedgerType, error) { return types, nil })(all) + WithLedgerWriters(func(string) (*bus.Conn, error) { return conn, nil })(all) + m := all.forModule("m").(*moduleBus) + + open := func(t *testing.T) *ledger.Ledger { + t.Helper() + l, err := all.ledger("m", "records") + if err != nil { + t.Fatal(err) + } + return l + } + apply := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) (buscalls.Applied, error) { + t.Helper() + return l.Apply(buscalls.ApplyRequest{Ledger: "records", Instance: instance, Operation: operation, Body: json.RawMessage(body)}, + buscalls.Author{Account: "m"}) + } + must := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) buscalls.Applied { + t.Helper() + got, err := apply(t, l, instance, operation, body) + if err != nil { + t.Fatal(err) + } + return got + } + held := func(t *testing.T, key string) ledger.Record { + t.Helper() + raw, _, err := open(t).State.Get(key) + if err != nil { + t.Fatal(err) + } + var r ledger.Record + if raw != nil { + _ = json.Unmarshal(raw, &r) + } + return r + } + snapshotIsState := func(t *testing.T, key string) { + t.Helper() + l := open(t) + state, _, _ := l.State.Get(key) + snap, _, _ := l.Snapshots.Get(key) + if string(snap) != string(state) { + t.Fatalf("the snapshot of %s is %s, its state %s", key, snap, state) + } + } + + cases := []struct { + name, key string + run func(t *testing.T) + }{ + {"two writers race and one is refused", "decision.race", func(t *testing.T) { + l := open(t) + l.History = &racing{History: l.History, before: func() { must(t, open(t), "decision/race", "note", `{"says":"first"}`) }} + _, err := apply(t, l, "decision/race", "note", `{"says":"second"}`) + if !errors.Is(err, ledger.ErrRaced) || !strings.Contains(err.Error(), "not taken") { + t.Fatalf("the racing writer was answered %v", err) + } + if got := held(t, "decision.race"); string(got.State) != `{"notes":[{"says":"first"}]}` { + t.Fatalf("the state is %s", got.State) + } + }}, + {"two interleaved applies leave the state at the higher sequence", "decision.interleave", func(t *testing.T) { + var inner buscalls.Applied + l := open(t) + l.History = &racing{History: l.History, after: func() { inner = must(t, open(t), "decision/interleave", "note", `{"says":"later"}`) }} + outer := must(t, l, "decision/interleave", "note", `{"says":"earlier"}`) + got := held(t, "decision.interleave") + if inner.Sequence <= outer.Sequence || got.Sequence != inner.Sequence || + string(got.State) != `{"notes":[{"says":"earlier"},{"says":"later"}]}` { + t.Fatalf("applied %d then %d, the state at %d: %s", outer.Sequence, inner.Sequence, got.Sequence, got.State) + } + }}, + {"a write cut between history and state is repaired by the next apply", "decision.cut", func(t *testing.T) { + must(t, open(t), "decision/cut", "note", `{"says":"one"}`) + l := open(t) + l.State = &cut{State: l.State, once: true} + if got := must(t, l, "decision/cut", "note", `{"says":"cut"}`); !got.Taken || !got.StateBehind { + t.Fatalf("the cut apply was answered %+v", got) + } + must(t, open(t), "decision/cut", "note", `{"says":"three"}`) + if got := held(t, "decision.cut"); string(got.State) != `{"notes":[{"says":"one"},{"says":"cut"},{"says":"three"}]}` { + t.Fatalf("the state is %s", got.State) + } + }}, + {"a full state bucket answers taken with its sequence and catches up", "decision.full", func(t *testing.T) { + must(t, open(t), "decision/full", "note", `{"says":"small"}`) + kv, _ := js.KeyValue(ctx, "m_records") + status, _ := kv.Status(ctx) + if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: int64(status.Bytes()) + 64}); err != nil { + t.Fatal(err) + } + got := must(t, open(t), "decision/full", "note", fmt.Sprintf(`{"says":%q}`, strings.Repeat("x", 512))) + if !got.Taken || got.Sequence == 0 || !got.StateBehind { + t.Fatalf("the full bucket's apply was answered %+v", got) + } + if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: -1}); err != nil { + t.Fatal(err) + } + after := must(t, open(t), "decision/full", "note", `{"says":"room"}`) + if st := held(t, "decision.full"); after.StateBehind || st.Sequence != after.Sequence || !strings.Contains(string(st.State), strings.Repeat("x", 512)) { + t.Fatalf("after room: %+v, the state at %d", after, st.Sequence) + } + }}, + {"an instance whose operations expired rebuilds from its snapshot", "decision.expired", func(t *testing.T) { + must(t, open(t), "decision/expired", "title", `{"title":"Kept"}`) + want := held(t, "decision.expired") + s, _ := js.Stream(ctx, "LEDGER_m_records") + if err := s.Purge(ctx, jetstream.WithPurgeSubject("mesh.ledger.m.records.decision.expired")); err != nil { + t.Fatal(err) + } + kv, _ := js.KeyValue(ctx, "m_records") + _ = kv.Purge(ctx, "decision.expired") + got, err := open(t).Rebuild(buscalls.RebuildRequest{Instance: "decision/expired", From: buscalls.RebuildFromSnapshot}) + if st := held(t, "decision.expired"); err != nil || got.Written != 1 || string(st.State) != string(want.State) { + t.Fatalf("rebuilt %+v, %v: the state %s, want %s", got, err, st.State, want.State) + } + }}, + {"rebuild, reindex and lag repair by hand", "decision.repair", func(t *testing.T) { + must(t, open(t), "decision/repair", "title", `{"title":"Repaired"}`) + kv, _ := js.KeyValue(ctx, "m_records") + _ = kv.Purge(ctx, "decision.repair") + l := open(t) + lag, err := l.Lag(buscalls.LedgerRequest{}) + if err != nil || !strings.Contains(fmt.Sprint(lag), "decision/repair") || len(lag.Indexes) == 0 { + t.Fatalf("lag says %+v, %v", lag, err) + } + if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/repair"}); err != nil || got.Written != 1 { + t.Fatalf("rebuilt %+v, %v", got, err) + } + if _, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil { + t.Fatal(err) + } + if lag, err := l.Lag(buscalls.LedgerRequest{}); err != nil || len(lag.Instances) != 0 || len(lag.Indexes) != 0 { + t.Fatalf("lag after the repairs says %+v, %v", lag, err) + } + q, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "Repaired.>"}) + if err != nil || len(q.Entries) != 1 { + t.Fatalf("queried %+v, %v", q, err) + } + }}, + {"a forged author is refused", "decision.forged", func(t *testing.T) { + _, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"decision/forged","operation":"title","body":{"title":"T","author":"op@shanks"},"call":"forged"}`)) + if err == nil || !strings.Contains(err.Error(), "the call forged, which is not a tool call in flight") { + t.Fatalf("a forged call was answered %v", err) + } + if last, _ := open(t).History.Last("decision.forged"); last != 0 { + t.Fatalf("the forged apply was appended at %d", last) + } + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.run(t) + if c.key != "decision.forged" { + snapshotIsState(t, c.key) + } + }) + } +} -- 2.54.0 From fadc73f73b487599ec19566981fc42c3e18968bd Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 15:15:00 +0100 Subject: [PATCH 08/12] Key and answer a ledger index by mesh-sdk's IndexKey and IndexEntry, the one shape the owner's watcher also writes (issue 1966 T3) --- node-tools/internal/bus/ledger.go | 7 ++--- node-tools/internal/ledger/memory.go | 9 ++---- node-tools/internal/ledger/read.go | 21 ++++++++++---- node-tools/internal/ledger/read_test.go | 10 +++---- node-tools/internal/ledger/repair.go | 37 +++++++++---------------- 5 files changed, 40 insertions(+), 44 deletions(-) diff --git a/node-tools/internal/bus/ledger.go b/node-tools/internal/bus/ledger.go index 5591379..417c64d 100644 --- a/node-tools/internal/bus/ledger.go +++ b/node-tools/internal/bus/ledger.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -290,7 +289,7 @@ func (c *Conn) LedgerIndexes(b membership.BusBinding) map[string]ledger.Index { return out } -func (s *busState) Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) { +func (s *busState) Entries(pattern string, limit int) ([]ledger.Indexed, error) { if pattern == "" { pattern = ">" } @@ -299,7 +298,7 @@ func (s *busState) Entries(pattern string, limit int) ([]buscalls.IndexEntry, er return nil, err } sort.Strings(keys) - var out []buscalls.IndexEntry + var out []ledger.Indexed for _, k := range keys { if limit > 0 && len(out) >= limit { break @@ -312,7 +311,7 @@ func (s *busState) Entries(pattern string, limit int) ([]buscalls.IndexEntry, er return out, err } if v != nil { - out = append(out, buscalls.IndexEntry{Key: k, Value: valueOf(v)}) + out = append(out, ledger.Indexed{Key: k, Value: v}) } } return out, nil diff --git a/node-tools/internal/ledger/memory.go b/node-tools/internal/ledger/memory.go index 240a225..0b14670 100644 --- a/node-tools/internal/ledger/memory.go +++ b/node-tools/internal/ledger/memory.go @@ -1,12 +1,9 @@ package ledger import ( - "encoding/json" "sort" "strings" "sync" - - "git.novox.be/novox/mesh-sdk/go/buscalls" ) // The memory stores keep what the bus keeps, as the bus answers it, for a test of a ledger without a bus: a stream whose @@ -166,7 +163,7 @@ type MemoryIndex struct { kv map[string][]byte } -func (x *MemoryIndex) Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) { +func (x *MemoryIndex) Entries(pattern string, limit int) ([]Indexed, error) { x.mu.Lock() defer x.mu.Unlock() if pattern == "" { @@ -179,12 +176,12 @@ func (x *MemoryIndex) Entries(pattern string, limit int) ([]buscalls.IndexEntry, } } sort.Strings(keys) - var out []buscalls.IndexEntry + var out []Indexed for _, k := range keys { if limit > 0 && len(out) >= limit { break } - out = append(out, buscalls.IndexEntry{Key: k, Value: json.RawMessage(x.kv[k])}) + out = append(out, Indexed{Key: k, Value: x.kv[k]}) } return out, nil } diff --git a/node-tools/internal/ledger/read.go b/node-tools/internal/ledger/read.go index 08c2e1c..87f4cea 100644 --- a/node-tools/internal/ledger/read.go +++ b/node-tools/internal/ledger/read.go @@ -16,13 +16,19 @@ import ( // 0597 §4). type Index interface { // Entries is the entries whose key a pattern names (`*` one name, `>` the rest; empty is every key), at most limit - // (every one when 0). - Entries(pattern string, limit int) ([]buscalls.IndexEntry, error) + // (every one when 0), each as the bucket holds it. + Entries(pattern string, limit int) ([]Indexed, error) // Put writes one entry; Delete removes one. Put(key string, value []byte) error Delete(key string) error } +// Indexed is one key of an index's bucket and its value, a buscalls.IndexEntry as JSON. +type Indexed struct { + Key string + Value []byte +} + // Watcher is a state bucket that hands each instance's record as it is and as it changes, under a key pattern. type Watcher interface { Watch(pattern string, deliver func(key string, value []byte, current bool) error) (func(), error) @@ -144,12 +150,17 @@ func (l *Ledger) Query(r buscalls.QueryRequest) (buscalls.QueryAnswer, error) { if err != nil { return buscalls.QueryAnswer{}, err } - entries, err := ix.Entries(r.Key, r.Limit) + held, err := ix.Entries(r.Key, r.Limit) if err != nil { return buscalls.QueryAnswer{}, err } - if entries == nil { - entries = []buscalls.IndexEntry{} + entries := make([]buscalls.IndexEntry, 0, len(held)) + for _, h := range held { + var e buscalls.IndexEntry + if err := json.Unmarshal(h.Value, &e); err != nil { + return buscalls.QueryAnswer{}, fmt.Errorf("the index %s's entry %s is not one: %w", r.Index, h.Key, err) + } + entries = append(entries, e) } return buscalls.QueryAnswer{Entries: entries}, nil } diff --git a/node-tools/internal/ledger/read_test.go b/node-tools/internal/ledger/read_test.go index 2539c63..755ca70 100644 --- a/node-tools/internal/ledger/read_test.go +++ b/node-tools/internal/ledger/read_test.go @@ -47,10 +47,10 @@ func TestReadsAndRepairsOfALedger(t *testing.T) { } }}, {"query reads a declared index by key, and an undeclared one is refused", func(t *testing.T, l *Ledger, f fakes) { - _ = f.index.Put("One.decision.1", []byte(`{"instance":"decision/1"}`)) - _ = f.index.Put("Two.decision.2", []byte(`{"instance":"decision/2"}`)) + _ = f.index.Put("One.decision.1", []byte(`{"instance":"decision/1","sequence":1}`)) + _ = f.index.Put("Two.decision.2", []byte(`{"instance":"decision/2","sequence":2}`)) got, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "One.>"}) - if err != nil || len(got.Entries) != 1 || string(got.Entries[0].Value) != `{"instance":"decision/1"}` { + if err != nil || len(got.Entries) != 1 || got.Entries[0] != (buscalls.IndexEntry{Instance: "decision/1", Sequence: 1}) { t.Fatalf("queried %+v, %v", got, err) } if _, err := l.Query(buscalls.QueryRequest{Index: "by-date"}); err == nil || !strings.Contains(err.Error(), `no index "by-date"`) { @@ -82,7 +82,7 @@ func TestReadsAndRepairsOfALedger(t *testing.T) { } }}, {"reindex builds a declared index from the states, and lag says an index behind until it does", func(t *testing.T, l *Ledger, f fakes) { - applied(t, l, "decision/1", "title", `{"title":"One ledger"}`, owner) + applied(t, l, "decision/1", "title", `{"title":"One"}`, owner) applied(t, l, "decision/2", "note", `{"says":"untitled"}`, owner) _ = f.index.Put("stale.decision.9", []byte(`{}`)) if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 1 || lag.Indexes[0].Behind != 1 { @@ -92,7 +92,7 @@ func TestReadsAndRepairsOfALedger(t *testing.T) { t.Fatalf("reindexed %+v, %v", got, err) } entries, _ := f.index.Entries("", 0) - if len(entries) != 1 || entries[0].Key != "One_ledger.decision.1" || string(entries[0].Value) != `{"instance":"decision/1","sequence":1}` { + if len(entries) != 1 || entries[0].Key != buscalls.IndexKey([]string{"One"}, "decision/1") || string(entries[0].Value) != `{"instance":"decision/1","sequence":1}` { t.Fatalf("the index holds %+v", entries) } if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 0 || len(lag.Instances) != 0 { diff --git a/node-tools/internal/ledger/repair.go b/node-tools/internal/ledger/repair.go index e0d2803..7221d79 100644 --- a/node-tools/internal/ledger/repair.go +++ b/node-tools/internal/ledger/repair.go @@ -3,7 +3,6 @@ package ledger import ( "encoding/json" "fmt" - "regexp" "sort" "strings" @@ -130,41 +129,33 @@ func (l *Ledger) Snapshot(r buscalls.InstanceRequest) (buscalls.Repaired, error) return out, nil } -// unsafe is what a key part cannot hold. -var unsafe = regexp.MustCompile(`[^-_A-Za-z0-9]`) - -// IndexEntryOf is an instance's entry in a declared index, as reindex writes it: its key the values of the fields the -// index is on, in their order, then the instance's key, each part with what a key cannot hold written as `_`; its value -// the instance and the sequence of the state it was taken from. False when the state holds none of those fields. -func IndexEntryOf(on []string, key string, rec Record) (string, []byte, bool) { +// indexEntryOf is an instance's entry in a declared index, as reindex writes it and the owner's watcher keeps it +// (mesh-sdk buscalls.IndexKey, buscalls.IndexEntry): its key the values of the fields the index is on, in their order, +// then the instance; its value the instance and the sequence of the state it was taken from. False when the state holds +// none of those fields. A value that is not text is written as its JSON. +func indexEntryOf(on []string, key string, rec Record) (string, []byte, bool) { var state map[string]any if json.Unmarshal(rec.State, &state) != nil || state == nil { return "", nil, false } - parts := make([]string, 0, len(on)+1) + values := make([]string, 0, len(on)) found := false for _, f := range on { v, has := state[f] - if !has { - parts = append(parts, "_") - continue - } - found = true + found = found || has s, ok := v.(string) if !ok { raw, _ := json.Marshal(v) s = string(raw) } - if s = unsafe.ReplaceAllString(s, "_"); s == "" { - s = "_" - } - parts = append(parts, s) + values = append(values, s) } if !found { return "", nil, false } - value, _ := json.Marshal(map[string]any{"instance": InstanceOf(key), "sequence": rec.Sequence}) - return strings.Join(append(parts, key), "."), value, true + instance := InstanceOf(key) + value, _ := json.Marshal(buscalls.IndexEntry{Instance: instance, Sequence: rec.Sequence}) + return buscalls.IndexKey(values, instance), value, true } // declared is the fields an index is on and the types declaring it. @@ -205,7 +196,7 @@ func (l *Ledger) wanted(index string) (map[string][]byte, map[string]uint64, err if err != nil { return nil, nil, err } - if k, v, ok := IndexEntryOf(on, key, held); ok { + if k, v, ok := indexEntryOf(on, key, held); ok { want[k], seqs[k] = v, held.Sequence } } @@ -295,9 +286,7 @@ func (l *Ledger) Lag(buscalls.LedgerRequest) (buscalls.Lagged, error) { } at := map[string]uint64{} for _, e := range held { - var v struct { - Sequence uint64 `json:"sequence"` - } + var v buscalls.IndexEntry _ = json.Unmarshal(e.Value, &v) at[e.Key] = v.Sequence } -- 2.54.0 From 5a9e997349b42c3a7bd0a3a11394ef7798e3c63a Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 15:15:58 +0100 Subject: [PATCH 09/12] Check a ledger operation against the types its binding carries, refusing a ledger bound with none (issue 1966 T2) --- .../runtime/ledger_credentials_test.go | 9 ++---- .../runtime/ledger_integration_test.go | 14 ++++----- node-tools/internal/runtime/ledgers.go | 29 +++++-------------- node-tools/internal/runtime/ledgers_test.go | 13 +++++---- node-tools/internal/runtime/runtime.go | 5 ---- 5 files changed, 25 insertions(+), 45 deletions(-) diff --git a/node-tools/internal/runtime/ledger_credentials_test.go b/node-tools/internal/runtime/ledger_credentials_test.go index a517f7c..0a801d9 100644 --- a/node-tools/internal/runtime/ledger_credentials_test.go +++ b/node-tools/internal/runtime/ledger_credentials_test.go @@ -67,7 +67,9 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { } mem := mt.MembershipOf("m", "anchor", false, nil) mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records", + Types: []catalogue.LedgerType{{Name: "decision", Operations: []catalogue.LedgerOperation{ + {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}}}}}}} raw, _ := json.Marshal(mem) var sdk membership.Membership _ = json.Unmarshal(raw, &sdk) @@ -96,12 +98,7 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { } creds := FollowLedgerCredentials(file, bus.Dial, t.Logf) t.Cleanup(creds.Stop) - types := func(string, string) ([]catalogue.LedgerType, error) { - return []catalogue.LedgerType{{Name: "decision", Operations: []catalogue.LedgerOperation{ - {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}}}}}, nil - } all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}} - WithLedgerTypes(types)(all) WithLedgerWriters(creds.Writer)(all) m := all.forModule("m").(*moduleBus) apply := func(instance string) error { diff --git a/node-tools/internal/runtime/ledger_integration_test.go b/node-tools/internal/runtime/ledger_integration_test.go index 11d8153..c58e304 100644 --- a/node-tools/internal/runtime/ledger_integration_test.go +++ b/node-tools/internal/runtime/ledger_integration_test.go @@ -78,10 +78,17 @@ func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { t.Fatal(err) } } + types := []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, + Operations: []catalogue.LedgerOperation{ + {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}, + {Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}}, + }, + Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}} binding := membership.BusBinding{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger-snapshot.m.records"} mem := mt.MembershipOf("m", "anchor", false, nil) + binding.Types = types mem.Bindings = []membership.BusBinding{binding} mesh.Issue(t, mem) conn := connect(t, "node-tools", "anchor") @@ -92,14 +99,7 @@ func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { } return nil }) - types := []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, - Operations: []catalogue.LedgerOperation{ - {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}, - {Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}}, - }, - Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}} all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}} - WithLedgerTypes(func(string, string) ([]catalogue.LedgerType, error) { return types, nil })(all) WithLedgerWriters(func(string) (*bus.Conn, error) { return conn, nil })(all) m := all.forModule("m").(*moduleBus) diff --git a/node-tools/internal/runtime/ledgers.go b/node-tools/internal/runtime/ledgers.go index 7c15a1b..b3d62dc 100644 --- a/node-tools/internal/runtime/ledgers.go +++ b/node-tools/internal/runtime/ledgers.go @@ -20,21 +20,9 @@ import ( // for the module whose bundle or whose repair tool asks, from its own bindings alone, with its types and the connection // its history is published on. -// LedgerTypes is where the runner reads a ledger's types, by module and local name. Where it reads them from on the -// live mesh is not decided (issue 1966: the binding carries no types, and no record names a source); until it is, -// the runner refuses to open a ledger and says why, and a test gives the types. -type LedgerTypes func(module, local string) ([]catalogue.LedgerType, error) - -// errTypesUndecided is the refusal of a ledger whose types the runner has no source for. -func errTypesUndecided(module, local string) error { - return fmt.Errorf("the tool runner cannot open %s's ledger %q: it reads a ledger's types from nowhere yet, since its "+ - "binding carries none and no record names their source (issue 1966)", module, local) -} - -// ledgerSource is how the runner opens a module's ledger: its types, and the connection its history and snapshots are -// published on (the module's ledger credential, ADR 0597 §9). +// ledgerSource is how the runner opens a module's ledger beside its binding: the connection its history and snapshots +// are published on (the module's ledger credential, ADR 0597 §9). type ledgerSource struct { - types LedgerTypes writer func(module string) (*bus.Conn, error) } @@ -97,13 +85,12 @@ func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) { if err != nil { return nil, err } - types := c.ledgers.types - if types == nil { - return nil, errTypesUndecided(module, local) - } - ts, err := types(module, local) - if err != nil { - return nil, err + // Its types travel on its binding, copied by the grants part from the entry the manifest-accepted event carries + // (mesh-sdk membership.BusBinding.Types): the runner reads no other part's state for them (ADR 0596). + ts := b.Types + if len(ts) == 0 { + return nil, fmt.Errorf("%s's ledger %q is bound with no types, so no operation can be checked against one: its "+ + "binding carries the types its manifest asks for in `requires` (novox/hq ADR 0597 §1)", module, local) } var writer *bus.Conn if c.ledgers.writer != nil { diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index 789cece..8b17dbf 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -79,7 +79,7 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { } // aLedger raises one ledger of module m on the bus as the bus module would create it, binds it in m's membership, and -// answers the runtime's buses for m and for other, its types given as a test gives them. +// answers the runtime's buses for m and for other; its types travel on its binding, and a second binding carries none. func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { t.Helper() mesh = mt.New(t) @@ -103,7 +103,11 @@ func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { } mem := mt.MembershipOf("m", "anchor", false, nil) mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records", + Types: []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, + Operations: []catalogue.LedgerOperation{{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}}}}}, + {Local: "untyped", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records", + Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} mesh.Issue(t, mem) mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil)) conn := connect(t, "node-tools", "anchor") @@ -116,10 +120,6 @@ func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { return nil }) all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}, ledgers: ledgerSource{ - types: func(string, string) ([]catalogue.LedgerType, error) { - return []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, - Operations: []catalogue.LedgerOperation{{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}}}}, nil - }, writer: func(string) (*bus.Conn, error) { return conn, nil }, }} return all.forModule("m").(*moduleBus), all.forModule("other").(*moduleBus), mesh @@ -138,6 +138,7 @@ func TestAnApplyIsStampedWithTheCallerTheRuntimeProved(t *testing.T) { {"1966/AC17 a call in flight stamps its proven caller", `{"ledger":"records","instance":"decision/1","operation":"title",` + forged + `,"call":"` + call + `","author":"forged"}`, "op@shanks", ""}, {"naming no call stamps the account alone", `{"ledger":"records","instance":"decision/2","operation":"title",` + forged + `}`, "", ""}, {"1966/AC18 another module's call is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"` + theirs + `"}`, "", "the call " + theirs + ", which is another module's"}, + {"a ledger bound with no types is refused, naming it", `{"ledger":"untyped","instance":"decision/3","operation":"title",` + forged + `}`, "", `m's ledger "untyped" is bound with no types`}, {"a call not in flight is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"c0ffee"}`, "", "the call c0ffee, which is not a tool call in flight"}, } { t.Run(c.name, func(t *testing.T) { diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index 7e4756e..cd76059 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -242,11 +242,6 @@ func WithLedgerWriters(writer func(module string) (*bus.Conn, error)) Option { return func(c *consumers) { c.ledgers.writer = writer } } -// WithLedgerTypes gives the runtime where it reads a ledger's types (ledgers.go). -func WithLedgerTypes(types LedgerTypes) Option { - return func(c *consumers) { c.ledgers.types = types } -} - // Run launches, binds and serves, and serves each module's bundle again when it appears, changes or // goes, with no restart. It answers a stop function. func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any), opts ...Option) (func(), error) { -- 2.54.0 From 961ec0ef96b1c830af3b71db88996c440d77854f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 15:17:12 +0100 Subject: [PATCH 10/12] Append and snapshot a ledger on mesh-sdk's ledger subjects, the ones its account is granted, refusing a binding naming others (issue 1966 T6) --- node-tools/internal/bus/ledger.go | 23 +++++++++++++------ .../runtime/ledger_credentials_test.go | 8 +++---- .../runtime/ledger_integration_test.go | 10 ++++---- node-tools/internal/runtime/ledgers.go | 5 +++- node-tools/internal/runtime/ledgers_test.go | 15 +++++++----- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/node-tools/internal/bus/ledger.go b/node-tools/internal/bus/ledger.go index 417c64d..c9080eb 100644 --- a/node-tools/internal/bus/ledger.go +++ b/node-tools/internal/bus/ledger.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -18,17 +19,25 @@ import ( ) // A ledger on the bus (novox/hq ADR 0597, design 25 § 12.4; issue 1966): its history a stream with a subject per -// instance under the binding's subject, its state a key-value bucket, its snapshots a subject per instance under the -// binding's snapshot subject, the latest only (the stream that holds them keeps one per subject, as the bus module +// instance under mesh-sdk's buscompose.LedgerHistorySubject, its state a key-value bucket, its snapshots a subject per +// instance under buscompose.LedgerSnapshotSubject, the latest only (the stream that holds them keeps one per subject, as the bus module // creates it). **Only the module's ledger connection publishes on its history and its snapshots** (ADR 0597 §9): what // is read, and the state, go on the runner's own. -// LedgerStores are the stores of one ledger binding: reads and the state on c, the history's appends and the snapshots' -// writes on writer, the module's ledger connection. -func (c *Conn) LedgerStores(b membership.BusBinding, writer *Conn) (ledger.History, ledger.State, ledger.Snapshots) { - return &busHistory{read: c, write: writer, stream: b.History, prefix: prefixOf(b.Subject)}, +// LedgerStores are the stores of a module's ledger binding: reads and the state on c, the history's appends and the +// snapshots' writes on writer, the module's ledger connection. The subjects are mesh-sdk's, the ones the grants part +// grants the module's ledger account byte for byte; a binding naming others is refused, naming both. +func (c *Conn) LedgerStores(module string, b membership.BusBinding, writer *Conn) (ledger.History, ledger.State, ledger.Snapshots, error) { + history, snapshot := buscompose.LedgerHistorySubject(module, b.Local), buscompose.LedgerSnapshotSubject(module, b.Local) + for _, named := range []struct{ what, bound, want string }{{"history", b.Subject, history}, {"snapshot", b.Snapshot, snapshot}} { + if named.bound != "" && prefixOf(named.bound) != named.want { + return nil, nil, nil, fmt.Errorf("%s's ledger %q is bound with the %s subject %s, and its ledger account is granted "+ + "%s alone (mesh-sdk buscompose); the binding is refused", module, b.Local, named.what, named.bound, named.want) + } + } + return &busHistory{read: c, write: writer, stream: b.History, prefix: history}, &busState{c: c, bucket: b.State}, - &busSnapshots{read: c, write: writer, prefix: prefixOf(b.Snapshot)} + &busSnapshots{read: c, write: writer, prefix: snapshot}, nil } // prefixOf is a binding's subject as the prefix an instance's key is added to: written with or without its `.>`. diff --git a/node-tools/internal/runtime/ledger_credentials_test.go b/node-tools/internal/runtime/ledger_credentials_test.go index 0a801d9..783cddb 100644 --- a/node-tools/internal/runtime/ledger_credentials_test.go +++ b/node-tools/internal/runtime/ledger_credentials_test.go @@ -25,7 +25,7 @@ import ( // (the runner's own account may not publish on m's ledger, as the grants part grants it); given it missing, apply is // refused naming it; a change of the file is taken without a restart. func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { - ledgerSubjects := []string{"mesh.ledger.m.records.>", "mesh.ledger-snapshot.m.records.>"} + ledgerSubjects := []string{buscompose.LedgerHistorySubject("m", "records") + ".>", buscompose.LedgerSnapshotSubject("m", "records") + ".>"} users := []*server.User{ {Username: "node-tools", Password: "pw", Permissions: &server.Permissions{ Publish: &server.SubjectPermission{Allow: []string{">"}, Deny: ledgerSubjects}}}, @@ -53,8 +53,8 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { js, _ := jetstream.New(admin) ctx := context.Background() for _, cfg := range []jetstream.StreamConfig{ - {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, - {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1}, } { if _, err := js.CreateStream(ctx, cfg); err != nil { t.Fatal(err) @@ -67,7 +67,7 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { } mem := mt.MembershipOf("m", "anchor", false, nil) mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records", + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records", Snapshot: "mesh.ledger.m.records.snapshot", Types: []catalogue.LedgerType{{Name: "decision", Operations: []catalogue.LedgerOperation{ {Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}}}}}}} raw, _ := json.Marshal(mem) diff --git a/node-tools/internal/runtime/ledger_integration_test.go b/node-tools/internal/runtime/ledger_integration_test.go index c58e304..802fb03 100644 --- a/node-tools/internal/runtime/ledger_integration_test.go +++ b/node-tools/internal/runtime/ledger_integration_test.go @@ -66,8 +66,8 @@ func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { js, _ := jetstream.New(nc) ctx := context.Background() for _, cfg := range []jetstream.StreamConfig{ - {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, - {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1}, } { if _, err := js.CreateStream(ctx, cfg); err != nil { t.Fatal(err) @@ -85,8 +85,8 @@ func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { }, Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}} binding := membership.BusBinding{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", - Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger-snapshot.m.records"} + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records", + Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger.m.records.snapshot"} mem := mt.MembershipOf("m", "anchor", false, nil) binding.Types = types mem.Bindings = []membership.BusBinding{binding} @@ -207,7 +207,7 @@ func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) { must(t, open(t), "decision/expired", "title", `{"title":"Kept"}`) want := held(t, "decision.expired") s, _ := js.Stream(ctx, "LEDGER_m_records") - if err := s.Purge(ctx, jetstream.WithPurgeSubject("mesh.ledger.m.records.decision.expired")); err != nil { + if err := s.Purge(ctx, jetstream.WithPurgeSubject("mesh.ledger.m.records.history.decision.expired")); err != nil { t.Fatal(err) } kv, _ := js.KeyValue(ctx, "m_records") diff --git a/node-tools/internal/runtime/ledgers.go b/node-tools/internal/runtime/ledgers.go index b3d62dc..ab2be1e 100644 --- a/node-tools/internal/runtime/ledgers.go +++ b/node-tools/internal/runtime/ledgers.go @@ -98,7 +98,10 @@ func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) { return nil, err } } - history, state, snapshots := c.conn.LedgerStores(b, writer) + history, state, snapshots, err := c.conn.LedgerStores(module, b, writer) + if err != nil { + return nil, err + } // The owning module's authenticated account: the user of the connection its history is published on. account := module if writer != nil && writer.Credential().User != "" { diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index 8b17dbf..b688711 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -24,8 +24,8 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { mesh := mt.New(t) owner := mt.MembershipOf("owner", "anchor", true, nil) owner.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_owner_records", Subject: "mesh.ledger.owner.records", State: "owner_records", - Snapshot: "mesh.ledger-snapshot.owner.records"}} + History: "LEDGER_owner_records", Subject: "mesh.ledger.owner.records.history", State: "owner_records", + Snapshot: "mesh.ledger.owner.records.snapshot"}} plain := mt.MembershipOf("plain", "anchor", true, nil) mesh.Issue(t, owner) mesh.Issue(t, plain) @@ -91,8 +91,8 @@ func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { js, _ := jetstream.New(nc) ctx := context.Background() for _, cfg := range []jetstream.StreamConfig{ - {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.>"}}, - {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger-snapshot.m.records.>"}, MaxMsgsPerSubject: 1}, + {Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}}, + {Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1}, } { if _, err := js.CreateStream(ctx, cfg); err != nil { t.Fatal(err) @@ -103,11 +103,13 @@ func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) { } mem := mt.MembershipOf("m", "anchor", false, nil) mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, - History: "LEDGER_m_records", Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records", + History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records", Snapshot: "mesh.ledger.m.records.snapshot", Types: []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}}, Operations: []catalogue.LedgerOperation{{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}}}}}, {Local: "untyped", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records", - Subject: "mesh.ledger.m.records", State: "m_records", Snapshot: "mesh.ledger-snapshot.m.records"}} + State: "m_records"}, + {Local: "elsewhere", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records", + Subject: "mesh.ledger.m.records.history", State: "m_records", Types: []catalogue.LedgerType{{Name: "decision"}}}} mesh.Issue(t, mem) mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil)) conn := connect(t, "node-tools", "anchor") @@ -139,6 +141,7 @@ func TestAnApplyIsStampedWithTheCallerTheRuntimeProved(t *testing.T) { {"naming no call stamps the account alone", `{"ledger":"records","instance":"decision/2","operation":"title",` + forged + `}`, "", ""}, {"1966/AC18 another module's call is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"` + theirs + `"}`, "", "the call " + theirs + ", which is another module's"}, {"a ledger bound with no types is refused, naming it", `{"ledger":"untyped","instance":"decision/3","operation":"title",` + forged + `}`, "", `m's ledger "untyped" is bound with no types`}, + {"a binding naming subjects its account is not granted is refused, naming both", `{"ledger":"elsewhere","instance":"decision/3","operation":"title",` + forged + `}`, "", "bound with the history subject mesh.ledger.m.records.history, and its ledger account is granted mesh.ledger.m.elsewhere.history alone"}, {"a call not in flight is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"c0ffee"}`, "", "the call c0ffee, which is not a tool call in flight"}, } { t.Run(c.name, func(t *testing.T) { -- 2.54.0 From df929b48b5cb23b31f1f473f663fb5b1a45cc3fb Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 15:29:47 +0100 Subject: [PATCH 11/12] Leave the own secret ledgers out of node-tools' manifest until the vault holds it on every machine, reading a missing file as no ledger credentials; grant the test's ledger account as the grants part does (issue 1966 T6) --- node-tools/internal/runtime/ledger_credentials.go | 4 +++- .../internal/runtime/ledger_credentials_test.go | 13 +++++++------ node-tools/module.json | 3 +-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/node-tools/internal/runtime/ledger_credentials.go b/node-tools/internal/runtime/ledger_credentials.go index 3abca81..5b26218 100644 --- a/node-tools/internal/runtime/ledger_credentials.go +++ b/node-tools/internal/runtime/ledger_credentials.go @@ -19,7 +19,9 @@ import ( // that module's own ledger connection**; everything else stays on the runner's own. // LedgersFile is where the runner reads its own secret `ledgers`: MESH_LEDGERS_FILE when set, else beside its broker -// credential, both in its directory mesh-state (node-tools' manifest: `${dir:mesh-state}/ledgers`). +// credential, both in its directory mesh-state (`${dir:mesh-state}/ledgers`). Its manifest declares the secret only once +// the vault holds it on every machine (B3 of issue 1966): until then a file not there means no ledger credentials, and +// every ledger write is refused naming the missing one. const LedgersFile = "MESH_LEDGERS_FILE" // LedgersFileOf is the path of the secret `ledgers` for a runtime whose broker credential is at broker. diff --git a/node-tools/internal/runtime/ledger_credentials_test.go b/node-tools/internal/runtime/ledger_credentials_test.go index 783cddb..c96afd5 100644 --- a/node-tools/internal/runtime/ledger_credentials_test.go +++ b/node-tools/internal/runtime/ledger_credentials_test.go @@ -29,8 +29,11 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { users := []*server.User{ {Username: "node-tools", Password: "pw", Permissions: &server.Permissions{ Publish: &server.SubjectPermission{Allow: []string{">"}, Deny: ledgerSubjects}}}, + // As the grants part composes a ledger account (mesh-grants buscompose ledgerPermissions): publish on its + // module's ledger histories and snapshots, subscribe its own inbox, nothing else. {Username: "ledger.anchor.m", Password: "pw", Permissions: &server.Permissions{ - Publish: &server.SubjectPermission{Allow: append([]string{"$JS.API.>", "_INBOX.>"}, ledgerSubjects...)}}}, + Publish: &server.SubjectPermission{Allow: ledgerSubjects}, + Subscribe: &server.SubjectPermission{Allow: []string{"_INBOX.ledger.anchor.m.>"}}}}, {Username: "admin", Password: "pw"}, } s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), @@ -92,10 +95,8 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { return nil }) + // Not there yet, as on a machine whose vault has not made it: no ledger credentials, and the runner serves on. file := filepath.Join(t.TempDir(), "ledgers") - if err := os.WriteFile(file, []byte(`{}`), 0o600); err != nil { - t.Fatal(err) - } creds := FollowLedgerCredentials(file, bus.Dial, t.Logf) t.Cleanup(creds.Stop) all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}} @@ -106,8 +107,8 @@ func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) { return err } - // Missing: refused, naming it. - if err := apply("decision/1"); err == nil || !strings.Contains(err.Error(), "m's ledger credential is missing from the tool runner's own secret ledgers") { + // Missing: refused, naming it and why. + if err := apply("decision/1"); err == nil || !strings.Contains(err.Error(), "m's ledger credential is missing from the tool runner's own secret ledgers (the file "+file+" is not there yet)") { t.Fatalf("an apply with no credential: %v", err) } // Written: taken without a restart, on m's connection (the runner's own account is denied m's ledger subjects, so an diff --git a/node-tools/module.json b/node-tools/module.json index e456563..e6cb6db 100644 --- a/node-tools/module.json +++ b/node-tools/module.json @@ -28,8 +28,7 @@ } }, "own-secrets": { - "broker": "${dir:mesh-state}/broker", - "ledgers": "${dir:mesh-state}/ledgers" + "broker": "${dir:mesh-state}/broker" }, "listens": [ { -- 2.54.0 From 43d3fba501d392bf5d0f0883700e4ab2bae5048c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 15:35:30 +0100 Subject: [PATCH 12/12] Pin the group's mesh-sdk head f8bd12e in place of the local replace (issue 1966) --- node-tools/go.mod | 4 +--- node-tools/go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node-tools/go.mod b/node-tools/go.mod index f2cc3df..c355438 100644 --- a/node-tools/go.mod +++ b/node-tools/go.mod @@ -8,7 +8,7 @@ require ( ) require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,5 +20,3 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/time v0.15.0 // indirect ) - -replace git.novox.be/novox/mesh-sdk/go => ../../mesh-sdk/go diff --git a/node-tools/go.sum b/node-tools/go.sum index 98dc72c..0111c9e 100644 --- a/node-tools/go.sum +++ b/node-tools/go.sum @@ -1,3 +1,5 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 h1:T66tg9T/1hHg6KN/bDScHPT41IHkxLAGGO8VJiAOlb8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0