mesh-catalog/modules/mesh-knowledge/cmd/mesh-knowledge/store.go

package main

// Where the records are kept (novox/hq ADR 0290 rule 7): a database in the store, provided to the module
// (pgstore.go). The rules of the knowledge base are written once, against this interface; memStore is the same
// interface in memory, for the tests of those rules and for the terminal's dry run.

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"
)

// ErrExists is a create refused because the record is already held.
var ErrExists = errors.New("the record is already held")

// ErrChanged is a write refused because the entry changed since it was read.
var ErrChanged = errors.New("the record was changed by another write since it was read")

// Hit is one place a question matched: a clause of a record (Clause > 0) or a record as a whole.
type Hit struct {
	Key    Key
	Clause int
	Rank   float64
}

// Inbound is a link held on another record that points at this one.
type Inbound struct {
	From Key
	Link Link
}

// Finding is one failure of a rule that judges the whole record set.
type Finding struct {
	Rule    string    `json:"rule"`
	Records []string  `json:"records"`
	Says    string    `json:"says"`
	At      time.Time `json:"at"`
}

// Store keeps the records, every version of each, and the findings of the whole-set rules.
type Store interface {
	// Get is one record's entry, or nil.
	Get(ctx context.Context, k Key) (*Entry, error)
	// All is every record's entry, in kind then number order.
	All(ctx context.Context) ([]*Entry, error)
	// Create writes a record that is not held, with its first version; ErrExists when it is.
	Create(ctx context.Context, v Version) error
	// Save writes a new version of a held record. current says whether it becomes the version served (else it
	// is the one pending over it); clearPending drops the pending one. revision is the entry's revision as read:
	// ErrChanged when it moved.
	Save(ctx context.Context, v Version, current, clearPending bool, revision int64) error
	// Versions is every version of a record, oldest first.
	Versions(ctx context.Context, k Key) ([]Version, error)
	// Search ranks the clauses and records of the served versions against a question.
	Search(ctx context.Context, question string, limit int) ([]Hit, error)
	// Inbound is every link of another served record that points at this one.
	Inbound(ctx context.Context, k Key) ([]Inbound, error)
	// SetFindings replaces the whole-set findings; Findings reads them.
	SetFindings(ctx context.Context, f []Finding) error
	Findings(ctx context.Context) ([]Finding, error)
	// Told records that the operator was told of something, once, by its key: a topic's clauses drafted, or a
	// version that ran out of review tries (novox/hq issue 497); false when it was already.
	Told(ctx context.Context, topic string) (bool, error)
	// ToldOf is every key told of that starts with prefix; Untold takes one away once what it told of is over.
	ToldOf(ctx context.Context, prefix string) ([]string, error)
	Untold(ctx context.Context, key string) error
}

// memStore is the Store in memory.
type memStore struct {
	mu       sync.Mutex
	entries  map[Key]*memEntry
	findings []Finding
	told     map[string]bool
}

type memEntry struct {
	versions []Version
	current  int
	pending  int
	revision int64
}

func newMemStore() *memStore {
	return &memStore{entries: map[Key]*memEntry{}, told: map[string]bool{}}
}

func (e *memEntry) entry() *Entry {
	out := &Entry{CurrentVersion: e.current, PendingVersion: e.pending, Versions: len(e.versions), Revision: e.revision}
	out.Current = e.versions[e.current-1].Record.Clone()
	if e.pending > 0 {
		out.Pending = e.versions[e.pending-1].Record.Clone()
	}
	return out
}

func (s *memStore) Get(_ context.Context, k Key) (*Entry, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	e := s.entries[k]
	if e == nil {
		return nil, nil
	}
	return e.entry(), nil
}

func (s *memStore) All(context.Context) ([]*Entry, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	keys := make([]Key, 0, len(s.entries))
	for k := range s.entries {
		keys = append(keys, k)
	}
	sortKeys(keys)
	out := make([]*Entry, 0, len(keys))
	for _, k := range keys {
		out = append(out, s.entries[k].entry())
	}
	return out, nil
}

func sortKeys(keys []Key) {
	sort.Slice(keys, func(i, j int) bool {
		if keys[i].Kind != keys[j].Kind {
			return keys[i].Kind < keys[j].Kind
		}
		return keys[i].Number < keys[j].Number
	})
}

func (s *memStore) Create(_ context.Context, v Version) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	k := v.Record.Key()
	if s.entries[k] != nil {
		return ErrExists
	}
	v.N = 1
	v.Record = v.Record.Clone()
	s.entries[k] = &memEntry{versions: []Version{v}, current: 1, revision: 1}
	return nil
}

func (s *memStore) Save(_ context.Context, v Version, current, clearPending bool, revision int64) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	e := s.entries[v.Record.Key()]
	if e == nil {
		return fmt.Errorf("%s is not held", v.Record.Key())
	}
	if e.revision != revision {
		return ErrChanged
	}
	v.N = len(e.versions) + 1
	v.Record = v.Record.Clone()
	e.versions = append(e.versions, v)
	if current {
		e.current = v.N
	} else {
		e.pending = v.N
	}
	if clearPending {
		e.pending = 0
	}
	e.revision++
	return nil
}

func (s *memStore) Versions(_ context.Context, k Key) ([]Version, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	e := s.entries[k]
	if e == nil {
		return nil, nil
	}
	out := make([]Version, len(e.versions))
	for i, v := range e.versions {
		v.Record = v.Record.Clone()
		out[i] = v
	}
	return out, nil
}

// Search in memory: the question's words, stemmed crudely, counted in each clause and each record's title, topic
// and sections; the title and a clause weigh more. Enough to test the answers' shape; the store's ranking is the
// database's full-text search.
func (s *memStore) Search(_ context.Context, question string, limit int) ([]Hit, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	keys := make([]Key, 0, len(s.entries))
	for k := range s.entries {
		keys = append(keys, k)
	}
	sortKeys(keys)
	var all []*Entry
	for _, k := range keys {
		all = append(all, s.entries[k].entry())
	}
	return searchEntries(all, question, limit), nil
}

// searchEntries ranks the served versions of entries against a question.
func searchEntries(entries []*Entry, question string, limit int) []Hit {
	words := questionWords(question)
	if len(words) == 0 {
		return nil
	}
	var hits []Hit
	score := func(text string) float64 {
		n := 0.0
		for _, w := range wordsOf(text) {
			for _, q := range words {
				if stem(w) == q {
					n++
				}
			}
		}
		return n
	}
	for _, e := range entries {
		r := e.Current
		k := r.Key()
		if r.Status == StatusReserved || r.Status == StatusWithdrawn || r.Status == StatusRetracted {
			continue
		}
		for _, c := range r.Clauses {
			if c.Status == ClauseRetracted {
				continue
			}
			if n := score(c.Text); n > 0 {
				hits = append(hits, Hit{Key: k, Clause: c.N, Rank: n})
			}
		}
		body := ""
		for _, sec := range r.Sections {
			body += " " + sec.Text
		}
		if n := 2*score(r.Title) + score(r.Topic) + score(body)/20; n > 0 {
			hits = append(hits, Hit{Key: k, Rank: n})
		}
	}
	sort.Slice(hits, func(i, j int) bool {
		if hits[i].Rank != hits[j].Rank {
			return hits[i].Rank > hits[j].Rank
		}
		if hits[i].Key != hits[j].Key {
			return hits[i].Key.Number > hits[j].Key.Number
		}
		return hits[i].Clause < hits[j].Clause
	})
	if limit > 0 && len(hits) > limit {
		hits = hits[:limit]
	}
	return hits
}

func (s *memStore) Inbound(_ context.Context, k Key) ([]Inbound, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var served []*Record
	for _, e := range s.entries {
		served = append(served, e.versions[e.current-1].Record)
	}
	return inboundOf(served, k), nil
}

// inboundOf is every link of another served record in served that points at k.
func inboundOf(served []*Record, k Key) []Inbound {
	var out []Inbound
	for _, r := range served {
		from := r.Key()
		for _, l := range r.Links {
			if to, ok := l.To.Record(); ok && to == k {
				out = append(out, Inbound{From: from, Link: l})
			}
		}
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].From != out[j].From {
			return out[i].From.Number < out[j].From.Number
		}
		return out[i].Link.String() < out[j].Link.String()
	})
	return out
}

func (s *memStore) SetFindings(_ context.Context, f []Finding) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.findings = append([]Finding(nil), f...)
	return nil
}

func (s *memStore) Findings(context.Context) ([]Finding, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return append([]Finding(nil), s.findings...), nil
}

func (s *memStore) ToldOf(_ context.Context, prefix string) ([]string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var out []string
	for k := range s.told {
		if strings.HasPrefix(k, prefix) {
			out = append(out, k)
		}
	}
	sort.Strings(out)
	return out, nil
}

func (s *memStore) Untold(_ context.Context, key string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	delete(s.told, key)
	return nil
}

func (s *memStore) Told(_ context.Context, topic string) (bool, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.told[topic] {
		return false, nil
	}
	s.told[topic] = true
	return true, nil
}

// questionWords are the words of a question that say what it is about: the words of asking itself ("what
// holds about … and why") are left out, and each is stemmed.
func questionWords(q string) []string {
	var out []string
	seen := map[string]bool{}
	for _, w := range wordsOf(q) {
		if askingWords[w] || len(w) < 3 {
			continue
		}
		st := stem(w)
		if !seen[st] {
			seen[st] = true
			out = append(out, st)
		}
	}
	return out
}

// askingWords say that something is asked, not what about.
var askingWords = map[string]bool{"what": true, "holds": true, "hold": true, "about": true, "and": true, "why": true,
	"how": true, "does": true, "the": true, "decided": true, "decide": true, "say": true, "says": true, "which": true,
	"when": true, "who": true, "for": true, "with": true, "this": true, "that": true, "are": true, "was": true}

func wordsOf(s string) []string {
	return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) })
}

// stem is a crude stem: a plural's s and a few endings off.
func stem(w string) string {
	for _, suf := range []string{"ing", "es", "ed", "s"} {
		if len(w) > len(suf)+3 && strings.HasSuffix(w, suf) {
			return strings.TrimSuffix(w, suf)
		}
	}
	return w
}