Its 26 `<Module>_<state>` bucket constants (candidates, manifests, verdicts, packages, taken, the control records, pins, pending and the rest), its consumers, findings and first views were named from the constant Module; they are now index.Bucket(cred.Module, <state>), and its words name the state where they named the bucket. Builds: design 47 G A-296 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
398 lines
14 KiB
Go
398 lines
14 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// Kept while named (design 47 G §675, §693, §695): the indexer keeps, in its old state section named, one key per
|
|
// fingerprint holding the set of `<node>.<module>` whose want (the seat mesh-wants' wanted) or running build (the node-engine
|
|
// seat's running) names it, written by compare-and-set and capped at 4 MiB (max-mib, §695). An entry object is removed only
|
|
// when no kept revision lists it and its set is empty. Both events are streams kept last per subject, read whole at the
|
|
// start and then followed (§556), so the sets are the same whenever the indexer started.
|
|
const (
|
|
// NamedState is the old state section of the sets of members by fingerprint.
|
|
NamedState = "named"
|
|
// NamedDropTool is the tool that removes one member from one fingerprint's set by hand (class act).
|
|
NamedDropTool = "indexer_named_drop"
|
|
// NamedWanted and NamedRunning name the two streams the sets are read from, for Named.ReadDone.
|
|
NamedWanted = "wanted"
|
|
NamedRunning = "running"
|
|
)
|
|
|
|
// NamedValue is one key of named: the fingerprint and the members that name it, sorted.
|
|
type NamedValue struct {
|
|
V int `json:"v"`
|
|
Fp string `json:"fp"`
|
|
Members []string `json:"members"`
|
|
}
|
|
|
|
// NamedKey is the key of a fingerprint in named, the fingerprint escaped as a key's name is (§73).
|
|
func NamedKey(fingerprint string) string { return keys.EscapeName(fingerprint) }
|
|
|
|
// NamedMember is the member of a node's module in a set.
|
|
func NamedMember(node, module string) string { return node + "." + module }
|
|
|
|
// namedAt is the fingerprint one source last named for a member, and when it said so.
|
|
type namedAt struct {
|
|
fp string
|
|
at time.Time
|
|
}
|
|
|
|
// Named keeps the sets. What each member's want and running last named is held in memory only, read again at every start from
|
|
// the two streams, so a member whose fingerprint moved while the indexer was away leaves the old one's set at the end of the
|
|
// start's read (Reconcile).
|
|
type Named struct {
|
|
// Store is the state named.
|
|
Store Store
|
|
// Changed is called, with no lock held, after a set lost a member: an entry object it kept may be removed now (§675); nil
|
|
// calls nothing.
|
|
Changed func(ctx context.Context)
|
|
Logf func(format string, args ...any)
|
|
|
|
mu sync.Mutex
|
|
want map[string]namedAt
|
|
run map[string]namedAt
|
|
gone map[string]time.Time // the want of a member removed (§693), when
|
|
reads map[string]bool // the streams read whole at the start
|
|
ready bool
|
|
}
|
|
|
|
// NewNamed is the sets over store.
|
|
func NewNamed(store Store, logf func(string, ...any)) *Named {
|
|
if logf == nil {
|
|
logf = func(string, ...any) {}
|
|
}
|
|
return &Named{Store: store, Logf: logf, want: map[string]namedAt{}, run: map[string]namedAt{}, gone: map[string]time.Time{}, reads: map[string]bool{}}
|
|
}
|
|
|
|
// Ready says the two streams were read whole and the sets reconciled with them: until then no entry object is removed, so a
|
|
// restart reads named whole before it removes anything (§675).
|
|
func (n *Named) Ready() bool {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
return n.ready
|
|
}
|
|
|
|
// HearWanted applies one wanted event, said on mesh.seat.mesh-wants.event.wanted.<node>.<module>: the want names its fingerprint,
|
|
// or, with removed, names none and leaves the member out of every set whatever its node last said running (§693).
|
|
func (n *Named) HearWanted(ctx context.Context, data []byte) error {
|
|
var e meshwants.WantedEvent
|
|
if err := json.Unmarshal(data, &e); err != nil {
|
|
return fmt.Errorf("a wanted that does not read: %w", err)
|
|
}
|
|
if e.Node == "" || e.Module == "" || e.Fp == "" {
|
|
return errors.New("a wanted names no node, module or fingerprint")
|
|
}
|
|
var at time.Time
|
|
if e.At != nil {
|
|
at = *e.At
|
|
}
|
|
return n.apply(ctx, NamedMember(e.Node, e.Module), true, e.Fp, at, e.Removed)
|
|
}
|
|
|
|
// HearRunning applies one running event: the build it names; a stopped one, or one with no fingerprint, names none (design 47
|
|
// G §365).
|
|
func (n *Named) HearRunning(ctx context.Context, data []byte) error {
|
|
var r nodeengine.Running
|
|
if err := json.Unmarshal(data, &r); err != nil {
|
|
return fmt.Errorf("a running that does not read: %w", err)
|
|
}
|
|
if r.Node == "" || r.Module == "" {
|
|
return errors.New("a running names no node or module")
|
|
}
|
|
fp := r.Fingerprint
|
|
if r.Stopped {
|
|
fp = ""
|
|
}
|
|
return n.apply(ctx, NamedMember(r.Node, r.Module), false, fp, r.At, false)
|
|
}
|
|
|
|
// apply changes what one source names for the member and writes the sets it moves; one at a time, so the two sources never
|
|
// write one set from a stale view of each other.
|
|
func (n *Named) apply(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) error {
|
|
n.mu.Lock()
|
|
moved, err := n.applyLocked(ctx, member, wanted, fp, at, removed)
|
|
n.mu.Unlock()
|
|
if moved && n.Changed != nil {
|
|
n.Changed(ctx)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (n *Named) applyLocked(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) (bool, error) {
|
|
src := n.run
|
|
if wanted {
|
|
src = n.want
|
|
}
|
|
old, had := src[member]
|
|
if had && at.Before(old.at) {
|
|
return false, nil // an event older than the one heard changes nothing
|
|
}
|
|
var touched []string
|
|
if had {
|
|
touched = append(touched, old.fp)
|
|
}
|
|
switch {
|
|
case removed:
|
|
// The node left or the module was unplaced: the want goes, and with it what the node last said running.
|
|
if r, ok := n.run[member]; ok {
|
|
touched = append(touched, r.fp)
|
|
delete(n.run, member)
|
|
}
|
|
delete(n.want, member)
|
|
n.gone[member] = at
|
|
touched = append(touched, fp)
|
|
case !n.gone[member].IsZero() && !at.After(n.gone[member]):
|
|
return false, nil // an event older than the removal of the want it belongs to
|
|
case fp == "":
|
|
delete(src, member)
|
|
default:
|
|
if wanted {
|
|
delete(n.gone, member)
|
|
}
|
|
src[member] = namedAt{fp: fp, at: at}
|
|
touched = append(touched, fp)
|
|
}
|
|
return n.writeSets(ctx, member, touched)
|
|
}
|
|
|
|
// holds says whether either source names fp for the member.
|
|
func (n *Named) holds(member, fp string) bool {
|
|
return n.want[member].fp == fp || n.run[member].fp == fp
|
|
}
|
|
|
|
// writeSets makes each fingerprint's set hold the member exactly when a source names it; it answers whether a set lost one.
|
|
func (n *Named) writeSets(ctx context.Context, member string, fps []string) (lost bool, err error) {
|
|
slices.Sort(fps)
|
|
fps = slices.Compact(fps)
|
|
var errs []error
|
|
for _, fp := range fps {
|
|
if fp == "" {
|
|
continue
|
|
}
|
|
in := n.holds(member, fp)
|
|
changed := false
|
|
err := write(ctx, n.Store, NamedKey(fp), func(old []byte, found bool) ([]byte, bool, error) {
|
|
var v NamedValue
|
|
if found {
|
|
if err := json.Unmarshal(old, &v); err != nil {
|
|
return nil, false, fmt.Errorf("%s.%s does not read: %w", NamedState, NamedKey(fp), err)
|
|
}
|
|
}
|
|
has := slices.Contains(v.Members, member)
|
|
changed = has != in
|
|
switch {
|
|
case in && !has:
|
|
v.Members = append(v.Members, member)
|
|
case !in && has:
|
|
v.Members = slices.DeleteFunc(v.Members, func(m string) bool { return m == member })
|
|
default:
|
|
return nil, false, nil
|
|
}
|
|
if len(v.Members) == 0 {
|
|
return nil, true, nil
|
|
}
|
|
slices.Sort(v.Members)
|
|
v.V, v.Fp = keys.Version, fp
|
|
data, err := json.Marshal(v)
|
|
return data, false, err
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
if changed && !in {
|
|
lost = true
|
|
}
|
|
}
|
|
return lost, errors.Join(errs...)
|
|
}
|
|
|
|
// ReadDone says the stream (NamedWanted or NamedRunning) was read whole at the start; with both read, every key is reconciled
|
|
// with what the two named, a member whose build moved while the indexer was away leaving its old set, and the sets are ready.
|
|
func (n *Named) ReadDone(ctx context.Context, stream string) error {
|
|
n.mu.Lock()
|
|
n.reads[stream] = true
|
|
if n.ready || !n.reads[NamedWanted] || !n.reads[NamedRunning] {
|
|
n.mu.Unlock()
|
|
return nil
|
|
}
|
|
lost, err := n.reconcileLocked(ctx)
|
|
if err == nil {
|
|
n.ready = true
|
|
}
|
|
n.mu.Unlock()
|
|
if (lost || err == nil) && n.Changed != nil {
|
|
n.Changed(ctx) // the sets are ready: an entry kept back until now may go
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (n *Named) reconcileLocked(ctx context.Context) (bool, error) {
|
|
all, err := n.Store.Keys(ctx)
|
|
if err != nil {
|
|
return false, fmt.Errorf("the state %s cannot be listed: %w", NamedState, err)
|
|
}
|
|
var errs []error
|
|
lost := false
|
|
for _, key := range all {
|
|
err := write(ctx, n.Store, key, func(old []byte, found bool) ([]byte, bool, error) {
|
|
var v NamedValue
|
|
if !found || json.Unmarshal(old, &v) != nil {
|
|
return nil, false, nil
|
|
}
|
|
kept := slices.DeleteFunc(slices.Clone(v.Members), func(m string) bool { return !n.holds(m, v.Fp) })
|
|
if len(kept) == len(v.Members) {
|
|
return nil, false, nil
|
|
}
|
|
lost = true
|
|
if len(kept) == 0 {
|
|
return nil, true, nil
|
|
}
|
|
v.Members = kept
|
|
data, err := json.Marshal(v)
|
|
return data, false, err
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("%s.%s: %w", NamedState, key, err))
|
|
}
|
|
}
|
|
return lost, errors.Join(errs...)
|
|
}
|
|
|
|
// Members is the set of a fingerprint, sorted; none is an empty list.
|
|
func (n *Named) Members(ctx context.Context, fingerprint string) ([]string, error) {
|
|
return NamedMembers(ctx, n.Store, fingerprint)
|
|
}
|
|
|
|
// NamedMembers reads a fingerprint's set from the state.
|
|
func NamedMembers(ctx context.Context, st Store, fingerprint string) ([]string, error) {
|
|
value, _, found, err := st.Get(ctx, NamedKey(fingerprint))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the state %s cannot be read: %w", NamedState, err)
|
|
}
|
|
if !found {
|
|
return nil, nil
|
|
}
|
|
var v NamedValue
|
|
if err := json.Unmarshal(value, &v); err != nil {
|
|
return nil, fmt.Errorf("%s.%s does not read: %w", NamedState, NamedKey(fingerprint), err)
|
|
}
|
|
return v.Members, nil
|
|
}
|
|
|
|
// IsNamed says whether a want or a running build names the fingerprint. An unreadable state names it, so no entry is removed on
|
|
// a read that failed.
|
|
func (n *Named) IsNamed(ctx context.Context, fingerprint string) (bool, error) {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
m, err := n.Members(ctx, fingerprint)
|
|
return err != nil || len(m) > 0, err
|
|
}
|
|
|
|
// Drop removes one member from one fingerprint's set by hand (indexer_named_drop), for a member a want or a running build
|
|
// named that the mesh no longer holds; the next event of that member is applied as any is. It answers whether the member was in
|
|
// the set.
|
|
func (n *Named) Drop(ctx context.Context, fingerprint, member string) (bool, error) {
|
|
n.mu.Lock()
|
|
dropped := false
|
|
err := write(ctx, n.Store, NamedKey(fingerprint), func(old []byte, found bool) ([]byte, bool, error) {
|
|
var v NamedValue
|
|
if !found {
|
|
return nil, false, nil
|
|
}
|
|
if err := json.Unmarshal(old, &v); err != nil {
|
|
return nil, false, err
|
|
}
|
|
if !slices.Contains(v.Members, member) {
|
|
return nil, false, nil
|
|
}
|
|
dropped = true
|
|
v.Members = slices.DeleteFunc(v.Members, func(m string) bool { return m == member })
|
|
if len(v.Members) == 0 {
|
|
return nil, true, nil
|
|
}
|
|
data, err := json.Marshal(v)
|
|
return data, false, err
|
|
})
|
|
if dropped && err == nil {
|
|
if n.want[member].fp == fingerprint {
|
|
delete(n.want, member)
|
|
}
|
|
if n.run[member].fp == fingerprint {
|
|
delete(n.run, member)
|
|
}
|
|
}
|
|
n.mu.Unlock()
|
|
if dropped && err == nil && n.Changed != nil {
|
|
n.Changed(ctx)
|
|
}
|
|
return dropped, err
|
|
}
|
|
|
|
// NamedDrop is indexer_named_drop answered: the member leaves the fingerprint's set, and an entry nothing names now may go.
|
|
func (ix *Indexer) NamedDrop(ctx context.Context, fingerprint, member, by, why string) TakenIn {
|
|
r := TakenIn{Module: "named " + fingerprint}
|
|
if ix.NamedSets == nil {
|
|
r.Answer = "the indexer keeps no state " + NamedState + ": nothing dropped"
|
|
return r
|
|
}
|
|
dropped, err := ix.NamedSets.Drop(ctx, fingerprint, member)
|
|
switch {
|
|
case err != nil:
|
|
r.Answer = member + " could not be dropped from " + fingerprint + ": " + err.Error()
|
|
case !dropped:
|
|
r.Answer = member + " is not in the set of " + fingerprint + ": nothing dropped"
|
|
default:
|
|
r.Stored = true
|
|
r.Answer = fmt.Sprintf("%s is dropped from the set of %s by %s (%s)", member, fingerprint, by, why)
|
|
}
|
|
ix.logf("%s", r.Answer)
|
|
return r
|
|
}
|
|
|
|
// NamedTools is indexer_named_drop (class act): the tools bundle writes an ask in takeins and the indexer, the state's owner,
|
|
// removes the member; record says the hand act in the hand-act log before it is asked.
|
|
func NamedTools(ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
|
|
return []stdio.Tool{{Name: NamedDropTool,
|
|
Description: "Remove one member (`<node>.<module>`) from the set of a fingerprint in the indexer's state named, which keeps the entry object of " +
|
|
"the fingerprint while a want or a running build names it (design 47 G §675): for a member the mesh no longer holds, " +
|
|
"which indexer_entries shows. A hand act, recorded through mesh-hand-acts before it is asked; an entry nothing lists or names " +
|
|
"any more is removed.",
|
|
Input: map[string]any{"type": "object", "required": []string{"fingerprint", "member", "why"}, "properties": map[string]any{
|
|
"fingerprint": map[string]any{"type": "string", "description": "the entry's fingerprint"},
|
|
"member": map[string]any{"type": "string", "description": "the member, <node>.<module>, as indexer_entries names it"},
|
|
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
|
fp, member, why := str("fingerprint"), str("member"), str("why")
|
|
if fp == "" || member == "" || why == "" {
|
|
return nil, errors.New("name the fingerprint (fingerprint) and the member (member) and say why (why): a drop is a hand act, recorded through mesh-hand-acts")
|
|
}
|
|
data, err := json.Marshal(TakeInAsk{V: 1, NamedFingerprint: fp, NamedMember: member, Drop: true, Why: why, By: stdio.CallerOf(args), At: now().UTC()})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := record("dropped "+member+" from the names of the entry "+fp+" by hand", why); err != nil {
|
|
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
|
|
}
|
|
a, err := ask("named."+NamedKey(fp)+"."+keys.EscapeName(member), data)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return a.Result, nil
|
|
}}}
|
|
}
|