A key-value delete only adds a marker: every value of the per-user keys of the lists older than the two kept stayed in the state user-list's stream, which reached 94% of its 64 MiB cap on 2026-11-08 (5,968 keys, 13,228 values). At the cap the bus refuses grants' next write of the sealed list and deliveries lock. The part now takes those keys away through a purger on the process's own JetStream connection, a publish to the key's own subject the account already writes on; the tools, with no purger, still delete. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5
406 lines
17 KiB
Go
406 lines
17 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/seal"
|
|
"github.com/nats-io/nats-server/v2/server"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"mesh-grants/internal/buscompose"
|
|
grantsPart "mesh-grants/internal/grants"
|
|
)
|
|
|
|
// theBusNodeFollowsTheSealedListUnderTheComposedGrants, run by TestTheGrantsPartWhole, defends novox/hq ADR 0547 clause 7 on a real server (issue 1945): with the list this composer writes, the grants part
|
|
// creates its object bucket and its state, puts the sealed list and names it in `user-list.sealed`; the node-engine of
|
|
// the node holding mesh-broker watches that one key, reads the object by its name and opens it with its key, and may
|
|
// neither list the bucket, nor read another key of the state, nor write; another node reads nothing of either.
|
|
func theBusNodeFollowsTheSealedListUnderTheComposedGrants(t *testing.T) {
|
|
records := buscompose.Records{
|
|
Nodes: []string{"novox", "ace"},
|
|
Assigned: map[string][]buscompose.Declared{
|
|
"novox": {{Module: buscompose.GrantsModule, State: []buscompose.Bucket{{Module: buscompose.GrantsModule, Name: "user-list"}}}},
|
|
"ace": {{Module: "plex", Reads: []string{buscompose.ComposedRead}},
|
|
{Module: "mesh-delivery-view", Reads: []string{buscompose.ComposedRead}, ReadsObjects: []string{"mesh-knowledge_sections"}},
|
|
{Module: "mesh-knowledge", Objects: []buscompose.ObjectBucket{{Module: "mesh-knowledge", Name: "sections", MaxMiB: 1, Restricted: true}}}},
|
|
},
|
|
}
|
|
// The bus module and the router as the part composes them from their own manifests (testdata: the catalogue's
|
|
// nats and messenger at the registry milestone's join): the bus module's keeper process with its own account.
|
|
fromManifests, gaps := grantsPart.Records(grantsPart.Inputs{Nodes: []string{"novox"},
|
|
Assigned: map[string][]string{"novox": {"messenger", "nats"}},
|
|
Manifests: map[string]catalogue.Manifest{"nats": manifestIn(t, "nats"), "messenger": manifestIn(t, "messenger")}})
|
|
if len(gaps) != 0 {
|
|
t.Fatalf("composing nats and messenger from their manifests: %+v", gaps)
|
|
}
|
|
records.Assigned["novox"] = append(records.Assigned["novox"], fromManifests.Assigned["novox"]...)
|
|
records.BusState = fromManifests.BusState
|
|
// The node-engine's state as the part's records carry it: every node's compositions (ADR 0593 §6).
|
|
records.Engine = &buscompose.Declared{Module: buscompose.EngineModule, State: buscompose.EngineComposedBuckets(records.Nodes)}
|
|
users, err := buscompose.Users(records)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := range users {
|
|
users[i].PasswordHash = "pw-" + users[i].Username()
|
|
}
|
|
accounts, err := buscompose.ComposeAccounts(users)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dir := t.TempDir()
|
|
conf := filepath.Join(dir, "server.conf")
|
|
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: \""+dir+"\" }\n"+accounts), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts, err := server.ProcessConfigFile(conf)
|
|
if err != nil {
|
|
t.Fatalf("the composed list is not the server's: %v", err)
|
|
}
|
|
opts.NoLog, opts.NoSigs = true, true
|
|
s, err := server.NewServer(opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
go s.Start()
|
|
if !s.ReadyForConnections(30 * time.Second) {
|
|
t.Fatal("the bus did not come up")
|
|
}
|
|
t.Cleanup(s.Shutdown)
|
|
dial := func(user string) *nats.Conn {
|
|
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw-"+user), nats.CustomInboxPrefix("_INBOX."+user))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", user, err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
return nc
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// The grants part: its state and its object bucket, the sealed list put and named.
|
|
grants, err := jetstream.New(dial("novox." + buscompose.GrantsModule))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Its state is made as the grants part makes a declared state, by its stream's name (CreateKeyValue would ask the
|
|
// account's info, which no module is granted).
|
|
if _, err := grants.CreateStream(ctx, jetstream.StreamConfig{Name: "KV_" + buscompose.GrantsUserListBucket,
|
|
Subjects: []string{"$KV." + buscompose.GrantsUserListBucket + ".>"}, MaxMsgsPerSubject: 20, AllowDirect: true,
|
|
AllowRollup: true, DenyDelete: true, Discard: jetstream.DiscardNew}); err != nil {
|
|
t.Fatalf("the grants part's state: %v", err)
|
|
}
|
|
kv, err := grants.KeyValue(ctx, buscompose.GrantsUserListBucket)
|
|
if err != nil {
|
|
t.Fatalf("the grants part binds its state: %v", err)
|
|
}
|
|
objects, err := grants.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: buscompose.GrantsUserListObjects})
|
|
if err != nil {
|
|
t.Fatalf("the grants part's object bucket: %v", err)
|
|
}
|
|
node, err := seal.GenerateKeyPair()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
list := []byte(accounts)
|
|
digest := grantsstate.UserListDigest(list)
|
|
object, err := grantsstate.SealUserList(list, node.PublicKey)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name := grantsstate.UserListObjectName(digest)
|
|
for range 2 { // put twice: a second put of a name purges its old chunks
|
|
if _, err := objects.PutBytes(ctx, name, object); err != nil {
|
|
t.Fatalf("the grants part put its object: %v", err)
|
|
}
|
|
}
|
|
value, err := grantsstate.EncodeGrantsUserList(grantsstate.GrantsUserList{Follow: grantsstate.FollowGrants, Digest: digest,
|
|
Object: name, SealedTo: grantsstate.SealingKeyID(node.PublicKey), Node: "novox", Revision: 1, Users: len(users)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kv.Put(ctx, buscompose.GrantsUserListKey, value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kv.Put(ctx, "list", []byte(`{"v":1}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Issue 1966: the grants part purges its sealed lists' keys under its composed grants: listed with their last operations,
|
|
// each purged with its value, its state's bytes read before and after.
|
|
purging := ownPurging{ctx: ctx, js: grants, module: buscompose.GrantsModule}
|
|
live, marked := grantsPart.SealedUserKey("sha256:d9", "a.mod-x"), grantsPart.SealedUserKey("sha256:d8", "a.mod-x")
|
|
if _, err := kv.Put(ctx, live, bytes.Repeat([]byte("x"), 4096)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kv.Put(ctx, marked, []byte(`{}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := kv.Delete(ctx, marked); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ops, err := purging.LastOps(grantsPart.StateUserList, grantsPart.KeySealedGrants+".>")
|
|
if err != nil || len(ops) != 2 || ops[live] || !ops[marked] {
|
|
t.Fatalf("the grants part lists its sealed keys with their last operations: %v, %v", ops, err)
|
|
}
|
|
before, err := purging.Bytes(grantsPart.StateUserList)
|
|
if err != nil {
|
|
t.Fatalf("the grants part reads its state's bytes: %v", err)
|
|
}
|
|
for _, k := range []string{live, marked} {
|
|
if err := purging.Purge(grantsPart.StateUserList, k); err != nil {
|
|
t.Fatalf("the grants part purges its own %s: %v", k, err)
|
|
}
|
|
}
|
|
if after, err := purging.Bytes(grantsPart.StateUserList); err != nil || after+2048 > before { // the value goes; a purge marker of the key stays
|
|
t.Errorf("a purge kept the value: %d bytes before, %d after (%v)", before, after, err)
|
|
}
|
|
if _, err := kv.Get(ctx, live); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
t.Errorf("the purged %s is still read: %v", live, err)
|
|
}
|
|
|
|
// The bus node's engine: the key watched, the object by name (as mesh-host's link reads a declaration's file).
|
|
busNode := dial("node.novox")
|
|
js, err := busNode.JetStream(nats.MaxWait(5 * time.Second))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys, err := js.KeyValue(buscompose.GrantsUserListBucket)
|
|
if err != nil {
|
|
t.Fatalf("the bus node binds the grants part's state: %v", err)
|
|
}
|
|
w, err := keys.Watch(buscompose.GrantsUserListKey)
|
|
if err != nil {
|
|
t.Fatalf("the bus node watches the key: %v", err)
|
|
}
|
|
defer w.Stop()
|
|
var got grantsstate.GrantsUserList
|
|
select {
|
|
case e := <-w.Updates():
|
|
if e == nil {
|
|
t.Fatal("the watch found no value")
|
|
}
|
|
if got, err = grantsstate.DecodeGrantsUserList(e.Value()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("the watch said nothing")
|
|
}
|
|
read := func(js nats.JetStreamContext, name string) ([]byte, error) {
|
|
meta, err := js.GetLastMsg(buscompose.GrantsUserListObjectStream, "$O."+buscompose.GrantsUserListObjects+".M."+
|
|
base64.URLEncoding.EncodeToString([]byte(name)), nats.DirectGet())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var info struct {
|
|
NUID string `json:"nuid"`
|
|
Size int `json:"size"`
|
|
}
|
|
if err := json.Unmarshal(meta.Data, &info); err != nil {
|
|
return nil, err
|
|
}
|
|
sub, err := js.SubscribeSync("$O."+buscompose.GrantsUserListObjects+".C."+info.NUID, nats.OrderedConsumer(),
|
|
nats.BindStream(buscompose.GrantsUserListObjectStream))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer sub.Unsubscribe()
|
|
var body []byte
|
|
for len(body) < info.Size {
|
|
m, err := sub.NextMsg(5 * time.Second)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body = append(body, m.Data...)
|
|
}
|
|
return body, nil
|
|
}
|
|
body, err := read(js, got.Object)
|
|
if err != nil {
|
|
t.Fatalf("the bus node reads the object by its name: %v", err)
|
|
}
|
|
opened, err := grantsstate.OpenUserList(body, got.Digest, func(b seal.Box) ([]byte, error) {
|
|
s, err := seal.Open(b, node.PrivateKey)
|
|
return []byte(s), err
|
|
})
|
|
if err != nil || !bytes.Equal(opened, list) {
|
|
t.Fatalf("the bus node opened %d bytes, %v", len(opened), err)
|
|
}
|
|
|
|
// And nothing beyond: no other key, no list of the bucket, no write.
|
|
if _, err := keys.Get("list"); err == nil {
|
|
t.Error("the bus node read another key of the grants part's state")
|
|
}
|
|
if _, err := js.StreamInfo(buscompose.GrantsUserListObjectStream); err == nil {
|
|
t.Error("the bus node read the object stream's info, which lists every name")
|
|
}
|
|
if _, err := js.Publish("$O."+buscompose.GrantsUserListObjects+".M.x", []byte("x")); err == nil {
|
|
t.Error("the bus node wrote the grants part's bucket")
|
|
}
|
|
// Another node reads nothing of it.
|
|
other, err := dial("node.ace").JetStream(nats.MaxWait(2 * time.Second))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := read(other, got.Object); err == nil {
|
|
t.Error("another node read the object")
|
|
}
|
|
if _, err := other.GetLastMsg("KV_"+buscompose.GrantsUserListBucket, "$KV."+buscompose.GrantsUserListBucket+"."+buscompose.GrantsUserListKey, nats.DirectGet()); err == nil {
|
|
t.Errorf("another node read the key: %v", err)
|
|
}
|
|
// Review round 1 (running F4; design 25 § 12.7): the bus module's keeper connects on its own process account as the
|
|
// part composed it from nats' manifest, reads the operator's warrant in the router's record of its ask, and is
|
|
// refused a write outside its grants. The router's state is made as the part makes a declared state, and written by
|
|
// the router's own account, keyed `<asker>.<ask id>`.
|
|
{
|
|
asksStream := jetstream.StreamConfig{Name: "KV_messenger_asks", Subjects: []string{"$KV.messenger_asks.>"}, MaxMsgsPerSubject: 5,
|
|
AllowDirect: true, AllowRollup: true, DenyDelete: true, Discard: jetstream.DiscardNew}
|
|
if _, err := grants.CreateStream(ctx, asksStream); err != nil {
|
|
t.Fatalf("the grants part creates the router's asks: %v", err)
|
|
}
|
|
router, err := jetstream.New(dial("novox.messenger"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
asks, err := router.KeyValue(ctx, "messenger_asks")
|
|
if err != nil {
|
|
t.Fatalf("the router binds its asks: %v", err)
|
|
}
|
|
if _, err := asks.Put(ctx, "nats.bus-1", []byte(`{"asker":"nats","ask":{"id":"bus-1"},"state":"answered","warrant":{"ask":"bus-1","asker":"nats","outcome":"chosen"}}`)); err != nil {
|
|
t.Fatalf("the router writes its record: %v", err)
|
|
}
|
|
keeper, err := process.Connect(process.Account{URL: s.ClientURL(), Node: "novox", Module: "nats", User: "novox.nats.keeper",
|
|
Password: "pw-novox.nats.keeper"})
|
|
if err != nil {
|
|
t.Fatalf("the keeper connects on its composed account: %v", err)
|
|
}
|
|
t.Cleanup(keeper.Close)
|
|
e, err := keeper.Ask("messenger_asks", "bus-1")
|
|
if err != nil || e == nil || !bytes.Contains(e.Value, []byte(`"outcome":"chosen"`)) {
|
|
t.Fatalf("the keeper reads its ask's warrant under its own account: %+v, %v", e, err)
|
|
}
|
|
if e, err := keeper.Ask("messenger_asks", "other"); err != nil || e != nil {
|
|
t.Errorf("the keeper reads %+v (%v) for an ask it has no record of", e, err)
|
|
}
|
|
raw := dial("novox.nats.keeper")
|
|
for _, subject := range []string{"$KV.messenger_asks.nats.bus-1", "$KV." + buscompose.GrantsUserListBucket + ".list"} {
|
|
reply, err := raw.Request(subject, []byte(`{}`), 500*time.Millisecond)
|
|
if err == nil && reply.Header.Get("Status") == "" {
|
|
t.Errorf("the keeper wrote %s, outside its grants: %s", subject, reply.Data)
|
|
}
|
|
}
|
|
if _, err := raw.Request("$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-config.x", nil, 500*time.Millisecond); err == nil {
|
|
t.Error("the keeper read another asker's record")
|
|
}
|
|
}
|
|
// A context of its own: the refusals above each wait out their bound.
|
|
ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
// ADR 0593 §6 (D14): the grants part creates each node's compositions; ace's engine writes its own and not novox's;
|
|
// the view, a named reader, finds every node's bucket by listing and reads it, writing nothing; plex reads none.
|
|
for _, node := range records.Nodes {
|
|
cfg := buscompose.BucketStreamConfig(buscompose.EngineComposedBuckets([]string{node})[0])
|
|
if _, err := grants.CreateStream(ctx, cfg); err != nil {
|
|
t.Fatalf("the grants part creates %s: %v", cfg.Name, err)
|
|
}
|
|
}
|
|
aceEngine, _ := dial("node.ace").JetStream(nats.MaxWait(2 * time.Second))
|
|
if _, err := aceEngine.Publish("$KV.composed-ace.ace.node", []byte(`{}`)); err != nil {
|
|
t.Errorf("ace's engine cannot write its own compositions: %v", err)
|
|
}
|
|
if _, err := aceEngine.Publish("$KV.composed-novox.novox.node", []byte(`{}`)); err == nil {
|
|
t.Error("ace's engine wrote novox's compositions")
|
|
}
|
|
view, err := jetstream.New(dial("ace.mesh-delivery-view"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var found []string
|
|
for name := range view.KeyValueStoreNames(ctx).Name() {
|
|
found = append(found, name)
|
|
}
|
|
if !slices.Contains(found, "composed-ace") || !slices.Contains(found, "composed-novox") {
|
|
t.Errorf("the view finds the buckets %v", found)
|
|
}
|
|
composed, err := view.KeyValue(ctx, "composed-ace")
|
|
if err != nil {
|
|
t.Fatalf("the view binds ace's compositions: %v", err)
|
|
}
|
|
if _, err := composed.Get(ctx, "ace.node"); err != nil {
|
|
t.Errorf("the view reads ace's own composition: %v", err)
|
|
}
|
|
refusedIn := func() context.Context { // a refused publish is answered by no one: it waits out a short bound
|
|
c, stop := context.WithTimeout(ctx, 2*time.Second)
|
|
t.Cleanup(stop)
|
|
return c
|
|
}
|
|
if _, err := composed.Put(refusedIn(), "ace.node", []byte(`{}`)); err == nil {
|
|
t.Error("the view wrote a composition")
|
|
}
|
|
plex, _ := dial("ace.plex").JetStream(nats.MaxWait(2 * time.Second))
|
|
if _, err := plex.GetLastMsg("KV_composed-ace", "$KV.composed-ace.ace.node", nats.DirectGet()); err == nil {
|
|
t.Error("plex, not a named reader, read a composition")
|
|
}
|
|
// ADR 0592 §5, §9: a module's own object bucket, created by the grants part, written (and an object replaced) by its
|
|
// owner alone, read by the module its grant names, read-only, and by no other.
|
|
sections := buscompose.ObjectBucket{Module: "mesh-knowledge", Name: "sections", MaxMiB: 1, Restricted: true}
|
|
if _, err := grants.CreateStream(ctx, buscompose.ObjectsStreamConfig(sections)); err != nil {
|
|
t.Fatalf("the grants part creates %s: %v", sections.Stream(), err)
|
|
}
|
|
owner, err := jetstream.New(dial("ace.mesh-knowledge"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ownStore, err := owner.ObjectStore(ctx, sections.Bucket())
|
|
if err != nil {
|
|
t.Fatalf("the owner binds its object bucket: %v", err)
|
|
}
|
|
for range 2 { // put twice: a second put of a name purges its old chunks
|
|
if _, err := ownStore.PutBytes(ctx, "s1", []byte("body")); err != nil {
|
|
t.Fatalf("the owner puts an object: %v", err)
|
|
}
|
|
}
|
|
readStore, err := view.ObjectStore(ctx, sections.Bucket())
|
|
if err != nil {
|
|
t.Fatalf("the granted reader binds the bucket: %v", err)
|
|
}
|
|
if got, err := readStore.GetBytes(ctx, "s1"); err != nil || string(got) != "body" {
|
|
t.Errorf("the granted reader reads %q, %v", got, err)
|
|
}
|
|
if _, err := readStore.PutBytes(refusedIn(), "s2", []byte("x")); err == nil {
|
|
t.Error("the granted reader wrote the bucket")
|
|
}
|
|
if _, err := plex.GetLastMsg(sections.Stream(), "$O."+sections.Bucket()+".M.x", nats.DirectGet()); err == nil {
|
|
t.Error("a module the grant does not name read the bucket")
|
|
}
|
|
}
|
|
|
|
// manifestIn is a module's manifest as testdata keeps it.
|
|
func manifestIn(t *testing.T, module string) catalogue.Manifest {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(filepath.Join("testdata", module+".module.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, err := catalogue.Decode(raw)
|
|
if err != nil {
|
|
t.Fatalf("%s's manifest: %v", module, err)
|
|
}
|
|
return m
|
|
}
|