package main // Where the records are kept (novox/hq ADR 0290 rule 7): a database in the store, provided to the module // (pgstore.go). The rules of the knowledge base are written once, against this interface; memStore is the same // interface in memory, for the tests of those rules and for the terminal's dry run. import ( "context" "errors" "fmt" "sort" "strings" "sync" "time" "unicode" ) // ErrExists is a create refused because the record is already held. var ErrExists = errors.New("the record is already held") // ErrChanged is a write refused because the entry changed since it was read. var ErrChanged = errors.New("the record was changed by another write since it was read") // Hit is one place a question matched: a clause of a record (Clause > 0) or a record as a whole. type Hit struct { Key Key Clause int Rank float64 } // Inbound is a link held on another record that points at this one. type Inbound struct { From Key Link Link } // Finding is one failure of a rule that judges the whole record set. type Finding struct { Rule string `json:"rule"` Records []string `json:"records"` Says string `json:"says"` At time.Time `json:"at"` } // Store keeps the records, every version of each, and the findings of the whole-set rules. type Store interface { // Get is one record's entry, or nil. Get(ctx context.Context, k Key) (*Entry, error) // All is every record's entry, in kind then number order. All(ctx context.Context) ([]*Entry, error) // Create writes a record that is not held, with its first version; ErrExists when it is. Create(ctx context.Context, v Version) error // Save writes a new version of a held record. current says whether it becomes the version served (else it // is the one pending over it); clearPending drops the pending one. revision is the entry's revision as read: // ErrChanged when it moved. Save(ctx context.Context, v Version, current, clearPending bool, revision int64) error // Versions is every version of a record, oldest first. Versions(ctx context.Context, k Key) ([]Version, error) // Search ranks the clauses and records of the served versions against a question. Search(ctx context.Context, question string, limit int) ([]Hit, error) // Inbound is every link of another served record that points at this one. Inbound(ctx context.Context, k Key) ([]Inbound, error) // SetFindings replaces the whole-set findings; Findings reads them. SetFindings(ctx context.Context, f []Finding) error Findings(ctx context.Context) ([]Finding, error) // Told records that the operator was told of something, once, by its key: a topic's clauses drafted, or a // version that ran out of review tries (novox/hq issue 497); false when it was already. Told(ctx context.Context, topic string) (bool, error) // ToldOf is every key told of that starts with prefix; Untold takes one away once what it told of is over. ToldOf(ctx context.Context, prefix string) ([]string, error) Untold(ctx context.Context, key string) error } // memStore is the Store in memory. type memStore struct { mu sync.Mutex entries map[Key]*memEntry findings []Finding told map[string]bool } type memEntry struct { versions []Version current int pending int revision int64 } func newMemStore() *memStore { return &memStore{entries: map[Key]*memEntry{}, told: map[string]bool{}} } func (e *memEntry) entry() *Entry { out := &Entry{CurrentVersion: e.current, PendingVersion: e.pending, Versions: len(e.versions), Revision: e.revision} out.Current = e.versions[e.current-1].Record.Clone() if e.pending > 0 { out.Pending = e.versions[e.pending-1].Record.Clone() } return out } func (s *memStore) Get(_ context.Context, k Key) (*Entry, error) { s.mu.Lock() defer s.mu.Unlock() e := s.entries[k] if e == nil { return nil, nil } return e.entry(), nil } func (s *memStore) All(context.Context) ([]*Entry, error) { s.mu.Lock() defer s.mu.Unlock() keys := make([]Key, 0, len(s.entries)) for k := range s.entries { keys = append(keys, k) } sortKeys(keys) out := make([]*Entry, 0, len(keys)) for _, k := range keys { out = append(out, s.entries[k].entry()) } return out, nil } func sortKeys(keys []Key) { sort.Slice(keys, func(i, j int) bool { if keys[i].Kind != keys[j].Kind { return keys[i].Kind < keys[j].Kind } return keys[i].Number < keys[j].Number }) } func (s *memStore) Create(_ context.Context, v Version) error { s.mu.Lock() defer s.mu.Unlock() k := v.Record.Key() if s.entries[k] != nil { return ErrExists } v.N = 1 v.Record = v.Record.Clone() s.entries[k] = &memEntry{versions: []Version{v}, current: 1, revision: 1} return nil } func (s *memStore) Save(_ context.Context, v Version, current, clearPending bool, revision int64) error { s.mu.Lock() defer s.mu.Unlock() e := s.entries[v.Record.Key()] if e == nil { return fmt.Errorf("%s is not held", v.Record.Key()) } if e.revision != revision { return ErrChanged } v.N = len(e.versions) + 1 v.Record = v.Record.Clone() e.versions = append(e.versions, v) if current { e.current = v.N } else { e.pending = v.N } if clearPending { e.pending = 0 } e.revision++ return nil } func (s *memStore) Versions(_ context.Context, k Key) ([]Version, error) { s.mu.Lock() defer s.mu.Unlock() e := s.entries[k] if e == nil { return nil, nil } out := make([]Version, len(e.versions)) for i, v := range e.versions { v.Record = v.Record.Clone() out[i] = v } return out, nil } // Search in memory: the question's words, stemmed crudely, counted in each clause and each record's title, topic // and sections; the title and a clause weigh more. Enough to test the answers' shape; the store's ranking is the // database's full-text search. func (s *memStore) Search(_ context.Context, question string, limit int) ([]Hit, error) { words := questionWords(question) if len(words) == 0 { return nil, nil } s.mu.Lock() defer s.mu.Unlock() var hits []Hit score := func(text string) float64 { n := 0.0 for _, w := range wordsOf(text) { for _, q := range words { if stem(w) == q { n++ } } } return n } for k, e := range s.entries { r := e.versions[e.current-1].Record if r.Status == StatusReserved || r.Status == StatusWithdrawn { continue } for _, c := range r.Clauses { if n := score(c.Text); n > 0 { hits = append(hits, Hit{Key: k, Clause: c.N, Rank: n}) } } body := "" for _, sec := range r.Sections { body += " " + sec.Text } if n := 2*score(r.Title) + score(r.Topic) + score(body)/20; n > 0 { hits = append(hits, Hit{Key: k, Rank: n}) } } sort.Slice(hits, func(i, j int) bool { if hits[i].Rank != hits[j].Rank { return hits[i].Rank > hits[j].Rank } if hits[i].Key != hits[j].Key { return hits[i].Key.Number > hits[j].Key.Number } return hits[i].Clause < hits[j].Clause }) if limit > 0 && len(hits) > limit { hits = hits[:limit] } return hits, nil } func (s *memStore) Inbound(_ context.Context, k Key) ([]Inbound, error) { s.mu.Lock() defer s.mu.Unlock() var out []Inbound for from, e := range s.entries { r := e.versions[e.current-1].Record for _, l := range r.Links { if to, ok := l.To.Record(); ok && to == k { out = append(out, Inbound{From: from, Link: l}) } } } sort.Slice(out, func(i, j int) bool { if out[i].From != out[j].From { return out[i].From.Number < out[j].From.Number } return out[i].Link.String() < out[j].Link.String() }) return out, nil } func (s *memStore) SetFindings(_ context.Context, f []Finding) error { s.mu.Lock() defer s.mu.Unlock() s.findings = append([]Finding(nil), f...) return nil } func (s *memStore) Findings(context.Context) ([]Finding, error) { s.mu.Lock() defer s.mu.Unlock() return append([]Finding(nil), s.findings...), nil } func (s *memStore) ToldOf(_ context.Context, prefix string) ([]string, error) { s.mu.Lock() defer s.mu.Unlock() var out []string for k := range s.told { if strings.HasPrefix(k, prefix) { out = append(out, k) } } sort.Strings(out) return out, nil } func (s *memStore) Untold(_ context.Context, key string) error { s.mu.Lock() defer s.mu.Unlock() delete(s.told, key) return nil } func (s *memStore) Told(_ context.Context, topic string) (bool, error) { s.mu.Lock() defer s.mu.Unlock() if s.told[topic] { return false, nil } s.told[topic] = true return true, nil } // questionWords are the words of a question that say what it is about: the words of asking itself ("what // holds about … and why") are left out, and each is stemmed. func questionWords(q string) []string { var out []string seen := map[string]bool{} for _, w := range wordsOf(q) { if askingWords[w] || len(w) < 3 { continue } st := stem(w) if !seen[st] { seen[st] = true out = append(out, st) } } return out } // askingWords say that something is asked, not what about. var askingWords = map[string]bool{"what": true, "holds": true, "hold": true, "about": true, "and": true, "why": true, "how": true, "does": true, "the": true, "decided": true, "decide": true, "say": true, "says": true, "which": true, "when": true, "who": true, "for": true, "with": true, "this": true, "that": true, "are": true, "was": true} func wordsOf(s string) []string { return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) } // stem is a crude stem: a plural's s and a few endings off. func stem(w string) string { for _, suf := range []string{"ing", "es", "ed", "s"} { if len(w) > len(suf)+3 && strings.HasSuffix(w, suf) { return strings.TrimSuffix(w, suf) } } return w }