Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/named.go
T
jschoubben cf93d30150 Names N3 (indexer): the indexer names its own buckets from its credential's module
Its 26 `<Module>_<state>` bucket constants (candidates, manifests, verdicts, packages, taken, the
control records, pins, pending and the rest), its consumers, findings and first views were named
from the constant Module; they are now index.Bucket(cred.Module, <state>), and its words name
the state where they named the bucket.

Builds: design 47 G A-296

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 19:18:56 +01:00

398 lines
14 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Kept while named (design 47 G §675, §693, §695): the indexer keeps, in its old state section named, one key per
// fingerprint holding the set of `<node>.<module>` whose want (the seat mesh-wants' wanted) or running build (the node-engine
// seat's running) names it, written by compare-and-set and capped at 4 MiB (max-mib, §695). An entry object is removed only
// when no kept revision lists it and its set is empty. Both events are streams kept last per subject, read whole at the
// start and then followed (§556), so the sets are the same whenever the indexer started.
const (
// NamedState is the old state section of the sets of members by fingerprint.
NamedState = "named"
// NamedDropTool is the tool that removes one member from one fingerprint's set by hand (class act).
NamedDropTool = "indexer_named_drop"
// NamedWanted and NamedRunning name the two streams the sets are read from, for Named.ReadDone.
NamedWanted = "wanted"
NamedRunning = "running"
)
// NamedValue is one key of named: the fingerprint and the members that name it, sorted.
type NamedValue struct {
V int `json:"v"`
Fp string `json:"fp"`
Members []string `json:"members"`
}
// NamedKey is the key of a fingerprint in named, the fingerprint escaped as a key's name is (§73).
func NamedKey(fingerprint string) string { return keys.EscapeName(fingerprint) }
// NamedMember is the member of a node's module in a set.
func NamedMember(node, module string) string { return node + "." + module }
// namedAt is the fingerprint one source last named for a member, and when it said so.
type namedAt struct {
fp string
at time.Time
}
// Named keeps the sets. What each member's want and running last named is held in memory only, read again at every start from
// the two streams, so a member whose fingerprint moved while the indexer was away leaves the old one's set at the end of the
// start's read (Reconcile).
type Named struct {
// Store is the state named.
Store Store
// Changed is called, with no lock held, after a set lost a member: an entry object it kept may be removed now (§675); nil
// calls nothing.
Changed func(ctx context.Context)
Logf func(format string, args ...any)
mu sync.Mutex
want map[string]namedAt
run map[string]namedAt
gone map[string]time.Time // the want of a member removed (§693), when
reads map[string]bool // the streams read whole at the start
ready bool
}
// NewNamed is the sets over store.
func NewNamed(store Store, logf func(string, ...any)) *Named {
if logf == nil {
logf = func(string, ...any) {}
}
return &Named{Store: store, Logf: logf, want: map[string]namedAt{}, run: map[string]namedAt{}, gone: map[string]time.Time{}, reads: map[string]bool{}}
}
// Ready says the two streams were read whole and the sets reconciled with them: until then no entry object is removed, so a
// restart reads named whole before it removes anything (§675).
func (n *Named) Ready() bool {
n.mu.Lock()
defer n.mu.Unlock()
return n.ready
}
// HearWanted applies one wanted event, said on mesh.seat.mesh-wants.event.wanted.<node>.<module>: the want names its fingerprint,
// or, with removed, names none and leaves the member out of every set whatever its node last said running (§693).
func (n *Named) HearWanted(ctx context.Context, data []byte) error {
var e meshwants.WantedEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a wanted that does not read: %w", err)
}
if e.Node == "" || e.Module == "" || e.Fp == "" {
return errors.New("a wanted names no node, module or fingerprint")
}
var at time.Time
if e.At != nil {
at = *e.At
}
return n.apply(ctx, NamedMember(e.Node, e.Module), true, e.Fp, at, e.Removed)
}
// HearRunning applies one running event: the build it names; a stopped one, or one with no fingerprint, names none (design 47
// G §365).
func (n *Named) HearRunning(ctx context.Context, data []byte) error {
var r nodeengine.Running
if err := json.Unmarshal(data, &r); err != nil {
return fmt.Errorf("a running that does not read: %w", err)
}
if r.Node == "" || r.Module == "" {
return errors.New("a running names no node or module")
}
fp := r.Fingerprint
if r.Stopped {
fp = ""
}
return n.apply(ctx, NamedMember(r.Node, r.Module), false, fp, r.At, false)
}
// apply changes what one source names for the member and writes the sets it moves; one at a time, so the two sources never
// write one set from a stale view of each other.
func (n *Named) apply(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) error {
n.mu.Lock()
moved, err := n.applyLocked(ctx, member, wanted, fp, at, removed)
n.mu.Unlock()
if moved && n.Changed != nil {
n.Changed(ctx)
}
return err
}
func (n *Named) applyLocked(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) (bool, error) {
src := n.run
if wanted {
src = n.want
}
old, had := src[member]
if had && at.Before(old.at) {
return false, nil // an event older than the one heard changes nothing
}
var touched []string
if had {
touched = append(touched, old.fp)
}
switch {
case removed:
// The node left or the module was unplaced: the want goes, and with it what the node last said running.
if r, ok := n.run[member]; ok {
touched = append(touched, r.fp)
delete(n.run, member)
}
delete(n.want, member)
n.gone[member] = at
touched = append(touched, fp)
case !n.gone[member].IsZero() && !at.After(n.gone[member]):
return false, nil // an event older than the removal of the want it belongs to
case fp == "":
delete(src, member)
default:
if wanted {
delete(n.gone, member)
}
src[member] = namedAt{fp: fp, at: at}
touched = append(touched, fp)
}
return n.writeSets(ctx, member, touched)
}
// holds says whether either source names fp for the member.
func (n *Named) holds(member, fp string) bool {
return n.want[member].fp == fp || n.run[member].fp == fp
}
// writeSets makes each fingerprint's set hold the member exactly when a source names it; it answers whether a set lost one.
func (n *Named) writeSets(ctx context.Context, member string, fps []string) (lost bool, err error) {
slices.Sort(fps)
fps = slices.Compact(fps)
var errs []error
for _, fp := range fps {
if fp == "" {
continue
}
in := n.holds(member, fp)
changed := false
err := write(ctx, n.Store, NamedKey(fp), func(old []byte, found bool) ([]byte, bool, error) {
var v NamedValue
if found {
if err := json.Unmarshal(old, &v); err != nil {
return nil, false, fmt.Errorf("%s.%s does not read: %w", NamedState, NamedKey(fp), err)
}
}
has := slices.Contains(v.Members, member)
changed = has != in
switch {
case in && !has:
v.Members = append(v.Members, member)
case !in && has:
v.Members = slices.DeleteFunc(v.Members, func(m string) bool { return m == member })
default:
return nil, false, nil
}
if len(v.Members) == 0 {
return nil, true, nil
}
slices.Sort(v.Members)
v.V, v.Fp = keys.Version, fp
data, err := json.Marshal(v)
return data, false, err
})
if err != nil {
errs = append(errs, err)
continue
}
if changed && !in {
lost = true
}
}
return lost, errors.Join(errs...)
}
// ReadDone says the stream (NamedWanted or NamedRunning) was read whole at the start; with both read, every key is reconciled
// with what the two named, a member whose build moved while the indexer was away leaving its old set, and the sets are ready.
func (n *Named) ReadDone(ctx context.Context, stream string) error {
n.mu.Lock()
n.reads[stream] = true
if n.ready || !n.reads[NamedWanted] || !n.reads[NamedRunning] {
n.mu.Unlock()
return nil
}
lost, err := n.reconcileLocked(ctx)
if err == nil {
n.ready = true
}
n.mu.Unlock()
if (lost || err == nil) && n.Changed != nil {
n.Changed(ctx) // the sets are ready: an entry kept back until now may go
}
return err
}
func (n *Named) reconcileLocked(ctx context.Context) (bool, error) {
all, err := n.Store.Keys(ctx)
if err != nil {
return false, fmt.Errorf("the state %s cannot be listed: %w", NamedState, err)
}
var errs []error
lost := false
for _, key := range all {
err := write(ctx, n.Store, key, func(old []byte, found bool) ([]byte, bool, error) {
var v NamedValue
if !found || json.Unmarshal(old, &v) != nil {
return nil, false, nil
}
kept := slices.DeleteFunc(slices.Clone(v.Members), func(m string) bool { return !n.holds(m, v.Fp) })
if len(kept) == len(v.Members) {
return nil, false, nil
}
lost = true
if len(kept) == 0 {
return nil, true, nil
}
v.Members = kept
data, err := json.Marshal(v)
return data, false, err
})
if err != nil {
errs = append(errs, fmt.Errorf("%s.%s: %w", NamedState, key, err))
}
}
return lost, errors.Join(errs...)
}
// Members is the set of a fingerprint, sorted; none is an empty list.
func (n *Named) Members(ctx context.Context, fingerprint string) ([]string, error) {
return NamedMembers(ctx, n.Store, fingerprint)
}
// NamedMembers reads a fingerprint's set from the state.
func NamedMembers(ctx context.Context, st Store, fingerprint string) ([]string, error) {
value, _, found, err := st.Get(ctx, NamedKey(fingerprint))
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", NamedState, err)
}
if !found {
return nil, nil
}
var v NamedValue
if err := json.Unmarshal(value, &v); err != nil {
return nil, fmt.Errorf("%s.%s does not read: %w", NamedState, NamedKey(fingerprint), err)
}
return v.Members, nil
}
// IsNamed says whether a want or a running build names the fingerprint. An unreadable state names it, so no entry is removed on
// a read that failed.
func (n *Named) IsNamed(ctx context.Context, fingerprint string) (bool, error) {
n.mu.Lock()
defer n.mu.Unlock()
m, err := n.Members(ctx, fingerprint)
return err != nil || len(m) > 0, err
}
// Drop removes one member from one fingerprint's set by hand (indexer_named_drop), for a member a want or a running build
// named that the mesh no longer holds; the next event of that member is applied as any is. It answers whether the member was in
// the set.
func (n *Named) Drop(ctx context.Context, fingerprint, member string) (bool, error) {
n.mu.Lock()
dropped := false
err := write(ctx, n.Store, NamedKey(fingerprint), func(old []byte, found bool) ([]byte, bool, error) {
var v NamedValue
if !found {
return nil, false, nil
}
if err := json.Unmarshal(old, &v); err != nil {
return nil, false, err
}
if !slices.Contains(v.Members, member) {
return nil, false, nil
}
dropped = true
v.Members = slices.DeleteFunc(v.Members, func(m string) bool { return m == member })
if len(v.Members) == 0 {
return nil, true, nil
}
data, err := json.Marshal(v)
return data, false, err
})
if dropped && err == nil {
if n.want[member].fp == fingerprint {
delete(n.want, member)
}
if n.run[member].fp == fingerprint {
delete(n.run, member)
}
}
n.mu.Unlock()
if dropped && err == nil && n.Changed != nil {
n.Changed(ctx)
}
return dropped, err
}
// NamedDrop is indexer_named_drop answered: the member leaves the fingerprint's set, and an entry nothing names now may go.
func (ix *Indexer) NamedDrop(ctx context.Context, fingerprint, member, by, why string) TakenIn {
r := TakenIn{Module: "named " + fingerprint}
if ix.NamedSets == nil {
r.Answer = "the indexer keeps no state " + NamedState + ": nothing dropped"
return r
}
dropped, err := ix.NamedSets.Drop(ctx, fingerprint, member)
switch {
case err != nil:
r.Answer = member + " could not be dropped from " + fingerprint + ": " + err.Error()
case !dropped:
r.Answer = member + " is not in the set of " + fingerprint + ": nothing dropped"
default:
r.Stored = true
r.Answer = fmt.Sprintf("%s is dropped from the set of %s by %s (%s)", member, fingerprint, by, why)
}
ix.logf("%s", r.Answer)
return r
}
// NamedTools is indexer_named_drop (class act): the tools bundle writes an ask in takeins and the indexer, the state's owner,
// removes the member; record says the hand act in the hand-act log before it is asked.
func NamedTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
return []stdio.Tool{{Name: NamedDropTool,
Description: "Remove one member (`<node>.<module>`) from the set of a fingerprint in the indexer's state named, which keeps the entry object of " +
"the fingerprint while a want or a running build names it (design 47 G §675): for a member the mesh no longer holds, " +
"which indexer_entries shows. A hand act, recorded through mesh-hand-acts before it is asked; an entry nothing lists or names " +
"any more is removed.",
Input: map[string]any{"type": "object", "required": []string{"fingerprint", "member", "why"}, "properties": map[string]any{
"fingerprint": map[string]any{"type": "string", "description": "the entry's fingerprint"},
"member": map[string]any{"type": "string", "description": "the member, <node>.<module>, as indexer_entries names it"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
fp, member, why := str("fingerprint"), str("member"), str("why")
if fp == "" || member == "" || why == "" {
return nil, errors.New("name the fingerprint (fingerprint) and the member (member) and say why (why): a drop is a hand act, recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, NamedFingerprint: fp, NamedMember: member, Drop: true, Why: why, By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
if err := record("dropped "+member+" from the names of the entry "+fp+" by hand", why); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
}
a, err := ask("named."+NamedKey(fp)+"."+keys.EscapeName(member), data)
if err != nil {
return nil, err
}
return a.Result, nil
}}}
}