Add mesh-knowledge, the knowledge base's first slice (hq issue 425, ADR 0290)

An agent learns what a decision says only by reading whole records. This module
holds every decision record and research effort whole in a database in the store,
with clauses that carry their own status and source, typed links read from both
ends, and every version. It answers ask, get and holds within a stated bound, and
takes a record through reserve, propose (refused on any of the 29 per-record
rules), review by someone other than the author, and acceptance, on the
operator's proven warrant when the record needs it. The 4 whole-set rules make
the module unhealthy when they fail. The import reads hq at a commit through the
records module, never overwrites, and changes nothing when run again. Prose
decisions get drafted clauses that a second judgement must confirm before they
are served.

modules/mesh-knowledge/cmd/mesh-knowledge/reference.go

@@ -0,0 +1,222 @@
+package main
+
+// What the rules judge a record against and this module does not hold: the reading order's topics
+// (02-DECISIONS/README.md) and the glossary's retired words (00-META/glossary.md), read from hq through the
+// records module, and the daemons' lines, a copy kept beside this code (daemon-lines, compared with the
+// catalogue's own copy by a test).
+
+import (
+	_ "embed"
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+//go:embed daemon-lines
+var daemonLinesFile string
+
+// Reference is what the rules read besides the records.
+type Reference struct {
+	Commit string
+	// Topics are the reading order's topics.
+	Topics []string
+	// Retired are the glossary's words retired in hq's prose (no scope, or hq).
+	Retired []string
+	// Lines are the daemons' lines: a daemon and a command, and whether it is a line only with an argument.
+	Lines []DaemonLine
+}
+
+// DaemonLine is one daemon's command a person never types.
+type DaemonLine struct {
+	Daemon, Command string
+	WithArgument    bool
+}
+
+var (
+	topicLine = regexp.MustCompile(`(?m)^\d+\.\s+\*\*([^*]+)\*\*\s+—\s+` + "`" + `topic:\s*([^` + "`" + `]+)` + "`")
+	notLine   = regexp.MustCompile(`(?m)^\s*(?:[-*]\s+)?\*Not:\*(.*)$`)
+	struck    = regexp.MustCompile(`~~([^~]+)~~(?:\s*\((hq|tools)\))?`)
+)
+
+// ReadingOrder is the topics the reading order writes, in order.
+func ReadingOrder(readme string) []string {
+	var out []string
+	for _, m := range topicLine.FindAllStringSubmatch(readme, -1) {
+		out = append(out, strings.TrimSpace(m[2]))
+	}
+	return out
+}
+
+// RetiredWords are the glossary's words retired in hq's prose: a struck word on a *Not:* line with no scope or
+// the scope hq; "a / b" is two words.
+func RetiredWords(glossary string) []string {
+	var out []string
+	for _, line := range notLine.FindAllStringSubmatch(glossary, -1) {
+		for _, m := range struck.FindAllStringSubmatch(line[1], -1) {
+			if m[2] == "tools" {
+				continue
+			}
+			for _, w := range strings.Split(m[1], " / ") {
+				if w = strings.TrimSpace(w); w != "" {
+					out = append(out, w)
+				}
+			}
+		}
+	}
+	return out
+}
+
+// DaemonLines reads the daemons' lines in the catalogue's format: `<daemon> <command>`, ` …` after a command
+// that is a line only with an argument after it.
+func DaemonLines(file string) []DaemonLine {
+	var out []DaemonLine
+	for _, line := range strings.Split(file, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		with := strings.HasSuffix(line, " …")
+		line = strings.TrimSuffix(line, " …")
+		d, c, ok := strings.Cut(line, " ")
+		if ok {
+			out = append(out, DaemonLine{Daemon: d, Command: c, WithArgument: with})
+		}
+	}
+	return out
+}
+
+// LoadReference reads the reading order and the glossary from hq through the records module.
+func LoadReference(records Records) (*Reference, error) {
+	commit, err := records.Standing()
+	if err != nil {
+		return nil, fmt.Errorf("hq cannot be read through the records module (%v)", err)
+	}
+	readme, _, err := records.Read("02-DECISIONS/README.md")
+	if err != nil {
+		return nil, fmt.Errorf("the reading order cannot be read (%v)", err)
+	}
+	glossary, _, err := records.Read("00-META/glossary.md")
+	if err != nil {
+		return nil, fmt.Errorf("the glossary cannot be read (%v)", err)
+	}
+	ref := &Reference{Commit: commit, Topics: ReadingOrder(readme), Retired: RetiredWords(glossary), Lines: DaemonLines(daemonLinesFile)}
+	if len(ref.Topics) == 0 {
+		return nil, fmt.Errorf("hq's 02-DECISIONS/README.md at %s writes no reading order", short(commit))
+	}
+	if len(ref.Retired) == 0 {
+		return nil, fmt.Errorf("hq's glossary at %s names no retired word: the words rule would pass on anything", short(commit))
+	}
+	return ref, nil
+}
+
+func short(commit string) string {
+	if len(commit) > 12 {
+		return commit[:12]
+	}
+	return commit
+}
+
+// ---- running prose ---------------------------------------------------------------------------------------
+
+var (
+	codeBlock  = regexp.MustCompile("(?ms)^\\s*```.*?^\\s*```")
+	comment    = regexp.MustCompile(`(?s)<!--.*?-->`)
+	blockQuote = regexp.MustCompile(`(?m)^\s*>.*$`)
+	codeSpan   = regexp.MustCompile("`[^`\n]+`")
+	linkTarget = regexp.MustCompile(`\]\([^)\s]*\)`)
+	strike     = regexp.MustCompile(`~~[^~\n]+~~`)
+	quoted     = regexp.MustCompile(`"[^"\n]*"`)
+	curly      = regexp.MustCompile(`“[^”]*”`)
+	single     = regexp.MustCompile(`‘[^’\n]*’`)
+)
+
+func blankOut(text string, res ...*regexp.Regexp) string {
+	for _, re := range res {
+		text = re.ReplaceAllStringFunc(text, func(m string) string {
+			return strings.Map(func(r rune) rune {
+				if r == '\n' {
+					return r
+				}
+				return ' '
+			}, m)
+		})
+	}
+	return text
+}
+
+// Prose is the running prose of a text, as hq's words check reads it: code, comments, block quotes, quotations,
+// struck-through words and link targets blanked.
+func Prose(text string) string {
+	return blankOut(text, codeBlock, comment, blockQuote, codeSpan, linkTarget, strike, quoted, curly, single)
+}
+
+// OwnWords is a text with what is not its own words blanked; code is kept, since a line to type is written there.
+func OwnWords(text string) string {
+	return blankOut(text, comment, blockQuote, linkTarget, strike, quoted, curly, single)
+}
+
+// wordPattern is a retired word matched whole and in any case, a space matching any run of space or hyphen, its
+// plural matched as the word.
+func wordPattern(word string) *regexp.Regexp {
+	parts := strings.Fields(word)
+	for i, p := range parts {
+		parts[i] = regexp.QuoteMeta(p)
+	}
+	return regexp.MustCompile(`(?i)` + strings.Join(parts, `[\s-]+`) + `(?:e?s)?`)
+}
+
+func isWordRune(r rune) bool {
+	return r == '-' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
+}
+
+// wholeMatches are the matches of re in text that stand alone: no letter, digit, - or _ on either side.
+func wholeMatches(re *regexp.Regexp, text string) []string {
+	var out []string
+	for _, loc := range re.FindAllStringIndex(text, -1) {
+		before, after := rune(' '), rune(' ')
+		if loc[0] > 0 {
+			before = []rune(text[:loc[0]])[len([]rune(text[:loc[0]]))-1]
+		}
+		if loc[1] < len(text) {
+			after = []rune(text[loc[1]:])[0]
+		}
+		if !isWordRune(before) && !isWordRune(after) {
+			out = append(out, text[loc[0]:loc[1]])
+		}
+	}
+	return out
+}
+
+// RetiredIn are the retired words a text uses in its running prose.
+func (ref *Reference) RetiredIn(text string) []string {
+	prose := Prose(text)
+	var out []string
+	for _, w := range ref.Retired {
+		for _, m := range wholeMatches(wordPattern(w), prose) {
+			out = append(out, strings.Join(strings.Fields(m), " "))
+		}
+	}
+	return out
+}
+
+// LinesIn are the daemons' lines a text writes.
+func (ref *Reference) LinesIn(text string) []string {
+	own := OwnWords(text)
+	var out []string
+	for _, l := range ref.Lines {
+		d, c := regexp.QuoteMeta(l.Daemon), regexp.QuoteMeta(l.Command)
+		var re *regexp.Regexp
+		if l.WithArgument {
+			re = regexp.MustCompile(`(?m)(?:^|` + "`" + `)[ \t]*(?:\$[ \t]*)?` + d + `\s+` + c + `\s+[^\s` + "`" + `]`)
+			if re.MatchString(own) {
+				out = append(out, l.Daemon+" "+l.Command)
+			}
+			continue
+		}
+		if len(wholeMatches(regexp.MustCompile(d+`\s+`+c), own)) > 0 {
+			out = append(out, l.Daemon+" "+l.Command)
+		}
+	}
+	return out
+}