package process import ( "context" "errors" "fmt" "sort" "strings" "sync" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" stdio "git.novox.be/novox/mesh-sdk/go" ) // **A state its owner keeps one bucket per node is read as one state** (novox/hq ADR 0593 ยง6: each node's engine // writes its own `composed-`, and the provisioner, the assigner and the view read them all). A reader names it as // its manifest's `reads` does (`.`, such as `mesh-host.composed`) with the prefix of its buckets // (`composed-`): each bucket `` holds keys whose first part is that node, so a key names its bucket. // Get reads one key from its node's bucket; Keys and Watch go over every bucket of the family the bus holds, and Watch // takes in a node's bucket created after it began, as the bus says it (its creation advisory), so a node added later is // heard with no timer. Reads only: the owner's nodes write, as their grants allow. // What a reader of a per-node state is granted beside each bucket's read (the grants part composes them): the listing // of bucket names (FamilyListSubject) and the creation of any stream (FamilyCreatedAdvisory), which says a name only. const ( FamilyListSubject = "$JS.API.STREAM.NAMES" FamilyCreatedAdvisory = "$JS.EVENT.ADVISORY.STREAM.CREATED.*" ) // NodeFamily is a per-node state read across every node's bucket. type NodeFamily struct { Read string Prefix string c *Conn } // ReadPerNode is another module's state kept one bucket per node, named read in the reader's manifest, its buckets // prefix followed by the node's name. func (c *Conn) ReadPerNode(read, prefix string) NodeFamily { return NodeFamily{Read: read, Prefix: prefix, c: c} } // checked refuses a family with no read name or prefix. func (f NodeFamily) checked() error { module, name, ok := strings.Cut(f.Read, ".") if !ok || module == "" || name == "" || strings.Contains(name, ".") { return fmt.Errorf("%q is not a state another module keeps, .", f.Read) } if f.Prefix == "" || strings.ContainsAny(f.Prefix, ".*> ") { return fmt.Errorf("the state %s names no prefix of its buckets, or one a bucket name cannot start with: %q", f.Read, f.Prefix) } return nil } // Node is the state of one node of the family: its bucket ``, bound as Read binds a bucket. func (f NodeFamily) Node(node string) State { module, name, _ := strings.Cut(f.Read, ".") return f.c.bound(module, name, f.Prefix+node, func() error { if err := f.checked(); err != nil { return err } if node == "" || strings.ContainsAny(node, ".*> ") { return fmt.Errorf("the state %s: %q is not a node a bucket is named for", f.Read, node) } return nil }) } // nodeOf is the node a key of the family names: its first part. func nodeOf(key string) (string, error) { node, _, ok := strings.Cut(key, ".") if !ok || node == "" { return "", fmt.Errorf("%q names no node: a key of a per-node state starts .", key) } return node, nil } // Nodes are the nodes whose bucket of the family the bus holds now, sorted: the names of the key-value buckets that // start with the prefix, the prefix taken off. func (f NodeFamily) Nodes() ([]string, error) { if err := f.checked(); err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) defer cancel() js, err := jetstream.New(f.c.nc) if err != nil { return nil, err } names := js.KeyValueStoreNames(ctx) var nodes []string for name := range names.Name() { if node, ok := strings.CutPrefix(name, f.Prefix); ok && node != "" { nodes = append(nodes, node) } } if err := names.Error(); err != nil && !errors.Is(err, jetstream.ErrEndOfData) { return nil, fmt.Errorf("the state %s: listing the buckets %s*: %w", f.Read, f.Prefix, err) } sort.Strings(nodes) return nodes, nil } // Get is a key's current value, read from the bucket of the node the key names, or nil when it has none or that // node has no bucket. func (f NodeFamily) Get(key string) (*stdio.StateEntry, error) { node, err := nodeOf(key) if err != nil { return nil, err } e, err := f.Node(node).Get(key) if errors.Is(err, jetstream.ErrBucketNotFound) { return nil, nil } return e, err } // Keys is every key with a current value in every bucket of the family, sorted. func (f NodeFamily) Keys() ([]string, error) { nodes, err := f.Nodes() if err != nil { return nil, err } keys := []string{} for _, node := range nodes { k, err := f.Node(node).Keys() if err != nil { return nil, err } keys = append(keys, k...) } sort.Strings(keys) return keys, nil } // Watch is State.Watch over every bucket of the family: handle hears the current values of every bucket the bus holds, // then every change, and the current values and changes of a bucket the bus creates later. Calls to handle are one at // a time. key narrows it as State.Watch's does, within each bucket; a key whose first part names a node watches only // that node's bucket, but is still told of no bucket that appears later for another node. func (f NodeFamily) Watch(key string, handle func(stdio.StateChange) error) error { if err := f.checked(); err != nil { return err } var mu sync.Mutex one := func(c stdio.StateChange) error { mu.Lock() defer mu.Unlock() return handle(c) } watched := map[string]bool{} var wmu sync.Mutex watch := func(node string) error { wmu.Lock() if watched[node] { wmu.Unlock() return nil } watched[node] = true wmu.Unlock() return f.Node(node).Watch(key, one) } if first, _, _ := strings.Cut(key, "."); key != "" && first != "*" && first != "**" { return watch(first) } // Heard before the buckets are listed, so a bucket created between the two is not missed. A subject's wildcard is a // whole token, so every stream's creation is heard (FamilyCreatedAdvisory) and the family's picked by name. sub, err := f.c.nc.Subscribe(FamilyCreatedAdvisory, func(m *nats.Msg) { bucket, isKV := strings.CutPrefix(m.Subject, strings.TrimSuffix(FamilyCreatedAdvisory, "*")+"KV_") if node, ok := strings.CutPrefix(bucket, f.Prefix); isKV && ok && node != "" { go func() { _ = watch(node) }() } }) if err != nil { return fmt.Errorf("the state %s: hearing new buckets %s*: %w", f.Read, f.Prefix, err) } nodes, err := f.Nodes() if err != nil { _ = sub.Unsubscribe() return err } for _, node := range nodes { if err := watch(node); err != nil { _ = sub.Unsubscribe() return err } } return nil }