Add mesh-knowledge, the knowledge base's first slice (hq issue 425, ADR 0290)

An agent learns what a decision says only by reading whole records. This module
holds every decision record and research effort whole in a database in the store,
with clauses that carry their own status and source, typed links read from both
ends, and every version. It answers ask, get and holds within a stated bound, and
takes a record through reserve, propose (refused on any of the 29 per-record
rules), review by someone other than the author, and acceptance, on the
operator's proven warrant when the record needs it. The 4 whole-set rules make
the module unhealthy when they fail. The import reads hq at a commit through the
records module, never overwrites, and changes nothing when run again. Prose
decisions get drafted clauses that a second judgement must confirm before they
are served.

modules/mesh-knowledge/cmd/mesh-knowledge/store.go

@@ -0,0 +1,319 @@
+package main
+
+// Where the records are kept (novox/hq ADR 0290 rule 7): a database in the store, provided to the module
+// (pgstore.go). The rules of the knowledge base are written once, against this interface; memStore is the same
+// interface in memory, for the tests of those rules and for the terminal's dry run.
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+)
+
+// ErrExists is a create refused because the record is already held.
+var ErrExists = errors.New("the record is already held")
+
+// ErrChanged is a write refused because the entry changed since it was read.
+var ErrChanged = errors.New("the record was changed by another write since it was read")
+
+// Hit is one place a question matched: a clause of a record (Clause > 0) or a record as a whole.
+type Hit struct {
+	Key    Key
+	Clause int
+	Rank   float64
+}
+
+// Inbound is a link held on another record that points at this one.
+type Inbound struct {
+	From Key
+	Link Link
+}
+
+// Finding is one failure of a rule that judges the whole record set.
+type Finding struct {
+	Rule    string    `json:"rule"`
+	Records []string  `json:"records"`
+	Says    string    `json:"says"`
+	At      time.Time `json:"at"`
+}
+
+// Store keeps the records, every version of each, and the findings of the whole-set rules.
+type Store interface {
+	// Get is one record's entry, or nil.
+	Get(ctx context.Context, k Key) (*Entry, error)
+	// All is every record's entry, in kind then number order.
+	All(ctx context.Context) ([]*Entry, error)
+	// Create writes a record that is not held, with its first version; ErrExists when it is.
+	Create(ctx context.Context, v Version) error
+	// Save writes a new version of a held record. current says whether it becomes the version served (else it
+	// is the one pending over it); clearPending drops the pending one. revision is the entry's revision as read:
+	// ErrChanged when it moved.
+	Save(ctx context.Context, v Version, current, clearPending bool, revision int64) error
+	// Versions is every version of a record, oldest first.
+	Versions(ctx context.Context, k Key) ([]Version, error)
+	// Search ranks the clauses and records of the served versions against a question.
+	Search(ctx context.Context, question string, limit int) ([]Hit, error)
+	// Inbound is every link of another served record that points at this one.
+	Inbound(ctx context.Context, k Key) ([]Inbound, error)
+	// SetFindings replaces the whole-set findings; Findings reads them.
+	SetFindings(ctx context.Context, f []Finding) error
+	Findings(ctx context.Context) ([]Finding, error)
+	// Told records that the operator was told about a topic's clauses, once; false when it was already.
+	Told(ctx context.Context, topic string) (bool, error)
+}
+
+// memStore is the Store in memory.
+type memStore struct {
+	mu       sync.Mutex
+	entries  map[Key]*memEntry
+	findings []Finding
+	told     map[string]bool
+}
+
+type memEntry struct {
+	versions []Version
+	current  int
+	pending  int
+	revision int64
+}
+
+func newMemStore() *memStore {
+	return &memStore{entries: map[Key]*memEntry{}, told: map[string]bool{}}
+}
+
+func (e *memEntry) entry() *Entry {
+	out := &Entry{CurrentVersion: e.current, PendingVersion: e.pending, Versions: len(e.versions), Revision: e.revision}
+	out.Current = e.versions[e.current-1].Record.Clone()
+	if e.pending > 0 {
+		out.Pending = e.versions[e.pending-1].Record.Clone()
+	}
+	return out
+}
+
+func (s *memStore) Get(_ context.Context, k Key) (*Entry, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	e := s.entries[k]
+	if e == nil {
+		return nil, nil
+	}
+	return e.entry(), nil
+}
+
+func (s *memStore) All(context.Context) ([]*Entry, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	keys := make([]Key, 0, len(s.entries))
+	for k := range s.entries {
+		keys = append(keys, k)
+	}
+	sortKeys(keys)
+	out := make([]*Entry, 0, len(keys))
+	for _, k := range keys {
+		out = append(out, s.entries[k].entry())
+	}
+	return out, nil
+}
+
+func sortKeys(keys []Key) {
+	sort.Slice(keys, func(i, j int) bool {
+		if keys[i].Kind != keys[j].Kind {
+			return keys[i].Kind < keys[j].Kind
+		}
+		return keys[i].Number < keys[j].Number
+	})
+}
+
+func (s *memStore) Create(_ context.Context, v Version) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	k := v.Record.Key()
+	if s.entries[k] != nil {
+		return ErrExists
+	}
+	v.N = 1
+	v.Record = v.Record.Clone()
+	s.entries[k] = &memEntry{versions: []Version{v}, current: 1, revision: 1}
+	return nil
+}
+
+func (s *memStore) Save(_ context.Context, v Version, current, clearPending bool, revision int64) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	e := s.entries[v.Record.Key()]
+	if e == nil {
+		return fmt.Errorf("%s is not held", v.Record.Key())
+	}
+	if e.revision != revision {
+		return ErrChanged
+	}
+	v.N = len(e.versions) + 1
+	v.Record = v.Record.Clone()
+	e.versions = append(e.versions, v)
+	if current {
+		e.current = v.N
+	} else {
+		e.pending = v.N
+	}
+	if clearPending {
+		e.pending = 0
+	}
+	e.revision++
+	return nil
+}
+
+func (s *memStore) Versions(_ context.Context, k Key) ([]Version, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	e := s.entries[k]
+	if e == nil {
+		return nil, nil
+	}
+	out := make([]Version, len(e.versions))
+	for i, v := range e.versions {
+		v.Record = v.Record.Clone()
+		out[i] = v
+	}
+	return out, nil
+}
+
+// Search in memory: the question's words, stemmed crudely, counted in each clause and each record's title, topic
+// and sections; the title and a clause weigh more. Enough to test the answers' shape; the store's ranking is the
+// database's full-text search.
+func (s *memStore) Search(_ context.Context, question string, limit int) ([]Hit, error) {
+	words := questionWords(question)
+	if len(words) == 0 {
+		return nil, nil
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var hits []Hit
+	score := func(text string) float64 {
+		n := 0.0
+		for _, w := range wordsOf(text) {
+			for _, q := range words {
+				if stem(w) == q {
+					n++
+				}
+			}
+		}
+		return n
+	}
+	for k, e := range s.entries {
+		r := e.versions[e.current-1].Record
+		if r.Status == StatusReserved || r.Status == StatusWithdrawn {
+			continue
+		}
+		for _, c := range r.Clauses {
+			if n := score(c.Text); n > 0 {
+				hits = append(hits, Hit{Key: k, Clause: c.N, Rank: n})
+			}
+		}
+		body := ""
+		for _, sec := range r.Sections {
+			body += " " + sec.Text
+		}
+		if n := 2*score(r.Title) + score(r.Topic) + score(body)/20; n > 0 {
+			hits = append(hits, Hit{Key: k, Rank: n})
+		}
+	}
+	sort.Slice(hits, func(i, j int) bool {
+		if hits[i].Rank != hits[j].Rank {
+			return hits[i].Rank > hits[j].Rank
+		}
+		if hits[i].Key != hits[j].Key {
+			return hits[i].Key.Number > hits[j].Key.Number
+		}
+		return hits[i].Clause < hits[j].Clause
+	})
+	if limit > 0 && len(hits) > limit {
+		hits = hits[:limit]
+	}
+	return hits, nil
+}
+
+func (s *memStore) Inbound(_ context.Context, k Key) ([]Inbound, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var out []Inbound
+	for from, e := range s.entries {
+		r := e.versions[e.current-1].Record
+		for _, l := range r.Links {
+			if to, ok := l.To.Record(); ok && to == k {
+				out = append(out, Inbound{From: from, Link: l})
+			}
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].From != out[j].From {
+			return out[i].From.Number < out[j].From.Number
+		}
+		return out[i].Link.String() < out[j].Link.String()
+	})
+	return out, nil
+}
+
+func (s *memStore) SetFindings(_ context.Context, f []Finding) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.findings = append([]Finding(nil), f...)
+	return nil
+}
+
+func (s *memStore) Findings(context.Context) ([]Finding, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]Finding(nil), s.findings...), nil
+}
+
+func (s *memStore) Told(_ context.Context, topic string) (bool, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.told[topic] {
+		return false, nil
+	}
+	s.told[topic] = true
+	return true, nil
+}
+
+// questionWords are the words of a question that say what it is about: the words of asking itself ("what
+// holds about … and why") are left out, and each is stemmed.
+func questionWords(q string) []string {
+	var out []string
+	seen := map[string]bool{}
+	for _, w := range wordsOf(q) {
+		if askingWords[w] || len(w) < 3 {
+			continue
+		}
+		st := stem(w)
+		if !seen[st] {
+			seen[st] = true
+			out = append(out, st)
+		}
+	}
+	return out
+}
+
+// askingWords say that something is asked, not what about.
+var askingWords = map[string]bool{"what": true, "holds": true, "hold": true, "about": true, "and": true, "why": true,
+	"how": true, "does": true, "the": true, "decided": true, "decide": true, "say": true, "says": true, "which": true,
+	"when": true, "who": true, "for": true, "with": true, "this": true, "that": true, "are": true, "was": true}
+
+func wordsOf(s string) []string {
+	return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) })
+}
+
+// stem is a crude stem: a plural's s and a few endings off.
+func stem(w string) string {
+	for _, suf := range []string{"ing", "es", "ed", "s"} {
+		if len(w) > len(suf)+3 && strings.HasSuffix(w, suf) {
+			return strings.TrimSuffix(w, suf)
+		}
+	}
+	return w
+}