Add mesh-knowledge, the knowledge base's first slice (hq issue 425, ADR 0290)

An agent learns what a decision says only by reading whole records. This module holds every decision record and research effort whole in a database in the store, with clauses that carry their own status and source, typed links read from both ends, and every version. It answers ask, get and holds within a stated bound, and takes a record through reserve, propose (refused on any of the 29 per-record rules), review by someone other than the author, and acceptance, on the operator's proven warrant when the record needs it. The 4 whole-set rules make the module unhealthy when they fail. The import reads hq at a commit through the records module, never overwrites, and changes nothing when run again. Prose decisions get drafted clauses that a second judgement must confirm before they are served.
2026-10-11 04:58:35 +02:00
parent 27cb95e98f
commit ba7b455719
50 changed files with 11883 additions and 0 deletions
@@ -0,0 +1,533 @@
+package main
+
+// The rules a record is held to (novox/hq ADR 0290 rule 4, to-be 55 §4 and §5).
+//
+// **29 rules judge one record or its prose**, and `propose` refuses a record that breaks any of them, naming every
+// one it breaks at once. They are hq's checks (00-META/checks/records.py, cycle.py, index.py, words.py) as they
+// judge one record, moved to the write: 25 on its structure, 4 on its prose. Each has an id, said in the refusal,
+// and one case in the table test (TestEveryPerRecordRuleRefusesAProposal).
+//
+// **4 rules judge the whole record set** (wholeSetRules) and run after every acceptance, and in the check
+// process every few minutes: a failure is kept as a finding, which the module's health answers, so the mesh
+// raises a condition naming the rule and the records. It does not undo the acceptance.
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// Rule is one rule that judges one record: its id and what it holds.
+type Rule struct {
+	ID    string
+	Holds string
+	// From is the hq check it comes from.
+	From string
+}
+
+// PerRecordRules are the 29, in the order they are judged.
+var PerRecordRules = []Rule{
+	{"kind", "a record is a decision or a research effort", "cycle.py"},
+	{"reserved", "its number was reserved for its kind and holds no other record", "cycle.py (duplicate numbers)"},
+	{"title", "it has a title, one line of at most 200 characters", "records.py numbering"},
+	{"topic", "a decision's topic is one of the reading order's", "records.py topics, index.py"},
+	{"date", "its date is a day, YYYY-MM-DD", "records.py insights, cycle.py deferrals"},
+	{"deciders", "a decision names who decided it", "records.py (deciders on every record)"},
+	{"research-status", "a research effort is active, graduated or abandoned", "cycle.py research"},
+	{"has-clause", "a decision has at least one clause", "records.py clauses"},
+	{"clause-numbering", "its clauses are numbered from 1, in order, each once", "records.py clauses"},
+	{"clause-lead", "every clause opens with a bold lead", "records.py clauses"},
+	{"clause-size", "every clause says something, in at most 2000 bytes", "records.py clauses"},
+	{"context", "a decision has a Context section", "the decision record's form"},
+	{"consequences", "a decision has a Consequences section", "the decision record's form"},
+	{"checked", "a decision says how it is checked", "AGENTS.md: a rule states how it is checked"},
+	{"overview", "a research effort has its overview", "cycle.py research"},
+	{"link-kind", "every link is of a known kind and goes out from the record or one of its clauses", "records.py links"},
+	{"link-target", "every link names a record, and a clause, that exist", "records.py links, rests-on"},
+	{"link-fits", "a link's kind fits its ends", "cycle.py research became, records.py supersession"},
+	{"self-link", "no link points at the record itself", "records.py supersession"},
+	{"rests-on", "a record it extends is accepted or proposed, never superseded or withdrawn", "records.py rests-on"},
+	{"supersedes-accepted", "a record it supersedes or changes is accepted", "records.py supersession, partial-supersession"},
+	{"changes-in-force", "a clause it changes is in force or changed, never superseded, proposed or drafted", "records.py partial-supersession"},
+	{"live-citation", "a superseded record it cites is cited beside the record that replaced it", "records.py live-citation"},
+	{"became", "a graduated research effort names what it became", "cycle.py research"},
+	{"text-links", "every link its text writes to a decision record or research effort names one the knowledge base holds", "records.py links"},
+	{"retired-word", "its prose uses no word the glossary retired", "words.py retired"},
+	{"daemon-line", "it writes no daemon's line", "words.py lines"},
+	{"deferral", "a paragraph that puts work off names the issue that tracks it", "cycle.py deferrals"},
+	{"insight-form", "a progressive insight its text mentions is in the dated, marked form", "records.py insights"},
+}
+
+// Refusal is one rule a record breaks, and how.
+type Refusal struct {
+	Rule string `json:"rule"`
+	Says string `json:"says"`
+}
+
+func (r Refusal) String() string { return r.Rule + ": " + r.Says }
+
+// Lookup is the served version of a record the knowledge base holds, or nil.
+type Lookup func(Key) *Record
+
+// MostClauseBytes is the most one clause may hold.
+const MostClauseBytes = 2000
+
+var (
+	isDay    = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
+	deferral = regexp.MustCompile(`(?i)until (?:it is |they are |the code (?:is )?)?renamed|until the code rename|` +
+		`\ba later rename|not renamed here|identifier to rename|left (?:for|to) (?:a )?later|not decided here|` +
+		`\bpostponed\b|future work|\bdeferred\b|later work|not built here|\ba later change`)
+	issueRef      = regexp.MustCompile(`(?i)tracked (?:by|in|as)\s+\[?issue (\d{3,5})|\[?issue (\d{3,5})\]?(?:\([^)]*\))?\s+tracks\b`)
+	insightPhrase = regexp.MustCompile(`(?i)progressive insights?`)
+	insightMarker = regexp.MustCompile(`\*\*Progressive insights?[ \t]*[—–-][ \t]*(\d{4}-\d{2}-\d{2})[^*\n]*\*\*`)
+)
+
+// decisionSection is the text of a record's Decision section.
+func decisionSection(r *Record) (string, bool) {
+	for _, s := range r.Sections {
+		if s.Name == "Decision" || strings.HasPrefix(s.Name, "Decision ") || strings.HasPrefix(s.Name, "Decision:") {
+			return s.Text, true
+		}
+	}
+	return "", false
+}
+
+func hasSection(r *Record, names ...string) bool {
+	for _, s := range r.Sections {
+		for _, n := range names {
+			if strings.EqualFold(strings.TrimSpace(s.Name), n) && strings.TrimSpace(s.Text) != "" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// allText is every word of a record an agent reads: its title, clauses and sections.
+func allText(r *Record) string {
+	var b strings.Builder
+	b.WriteString(r.Title + "\n\n")
+	for _, c := range r.Clauses {
+		b.WriteString(c.Text + "\n\n")
+	}
+	for _, s := range r.Sections {
+		b.WriteString(s.Text + "\n\n")
+	}
+	return b.String()
+}
+
+// JudgeRecord holds a proposed record to the 29 rules. held is the entry its number holds (a reservation, the
+// proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
+func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
+	var out []Refusal
+	refuse := func(rule, format string, a ...any) {
+		out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
+	}
+	decision := r.Kind == KindDecision
+	if r.Kind != KindDecision && r.Kind != KindResearch {
+		refuse("kind", "%q is neither decision nor research", r.Kind)
+	}
+	switch {
+	case held == nil:
+		refuse("reserved", "%s was never reserved: reserve gives the number", r.Key())
+	case held.Current.Status == StatusWithdrawn:
+		refuse("reserved", "%s was reserved and withdrawn: reserve a new number", r.Key())
+	}
+	title := strings.TrimSpace(r.Title)
+	if title == "" || strings.Contains(title, "\n") || len([]rune(title)) > 200 {
+		refuse("title", "the title is empty, more than one line, or longer than 200 characters")
+	}
+	if decision && !contains(ref.Topics, r.Topic) {
+		refuse("topic", "%q is not a topic of the reading order: %s", r.Topic, strings.Join(ref.Topics, ", "))
+	}
+	if !isDay.MatchString(r.Date) {
+		refuse("date", "%q is not a day written YYYY-MM-DD", r.Date)
+	}
+	if decision && len(r.Deciders) == 0 {
+		refuse("deciders", "a decision names who decided it")
+	}
+	if !decision && !contains(ResearchStatuses, r.Effort) {
+		refuse("research-status", "%q is not active, graduated or abandoned", r.Effort)
+	}
+	if decision && len(r.Clauses) == 0 {
+		refuse("has-clause", "a decision is made of clauses, and this one has none")
+	}
+	for i, c := range r.Clauses {
+		if c.N != i+1 {
+			refuse("clause-numbering", "clause %d stands %d. in order: clauses are numbered 1, 2, 3 … each once", c.N, i+1)
+		}
+		if c.Lead() == "" {
+			refuse("clause-lead", "clause %d does not open with a bold lead (**What is decided.** …)", c.N)
+		}
+		if strings.TrimSpace(c.Text) == "" || len(c.Text) > MostClauseBytes {
+			refuse("clause-size", "clause %d is empty or longer than %d bytes", c.N, MostClauseBytes)
+		}
+	}
+	if decision && !hasSection(r, "Context") {
+		refuse("context", "a decision has a Context section: why it was decided")
+	}
+	if decision && !hasSection(r, "Consequences") {
+		refuse("consequences", "a decision has a Consequences section: what follows from it, and what got harder")
+	}
+	if decision && !hasSection(r, "How it is checked") {
+		refuse("checked", "a decision has a section \"How it is checked\": a rule says how it is verified")
+	}
+	if !decision && !hasSection(r, "00-overview.md", "overview") {
+		refuse("overview", "a research effort has its overview, a section named 00-overview.md")
+	}
+	self := r.Key()
+	superseders := map[Key]bool{}
+	for _, l := range r.Links {
+		if to, ok := l.To.Record(); ok && l.Kind == LinkSupersedes {
+			superseders[to] = true
+		}
+	}
+	cited := map[Key]bool{}
+	for _, l := range r.Links {
+		if to, ok := l.To.Record(); ok {
+			cited[to] = true
+		}
+	}
+	for _, l := range r.Links {
+		if !contains(LinkKinds, l.Kind) {
+			refuse("link-kind", "%q is not a kind of link: %s", l.Kind, strings.Join(LinkKinds, ", "))
+			continue
+		}
+		if l.From > 0 && r.Clause(l.From) == nil {
+			refuse("link-kind", "the link %s goes out from clause %d, which the record does not have", l, l.From)
+		}
+		to, isRecord := l.To.Record()
+		if isRecord && to == self {
+			refuse("self-link", "the link %s points at the record itself", l)
+			continue
+		}
+		var target *Record
+		if isRecord {
+			target = lookup(to)
+			if target == nil || target.Status == StatusReserved || target.Status == StatusWithdrawn {
+				refuse("link-target", "the link %s names %s, which the knowledge base does not hold", l, to)
+				continue
+			}
+			if l.To.Clause > 0 && target.Clause(l.To.Clause) == nil {
+				refuse("link-target", "the link %s names clause %d, which %s does not have", l, l.To.Clause, to)
+				continue
+			}
+		}
+		switch l.Kind {
+		case LinkBecame:
+			if r.Kind != KindResearch || !(l.To.Kind == KindDecision || l.To.Kind == TargetDesign) || l.To.Clause > 0 {
+				refuse("link-fits", "became goes from a research effort to a decision record or a design: not %s", l)
+			}
+		case LinkChanges:
+			if !decision || l.To.Kind != KindDecision || l.To.Clause == 0 {
+				refuse("link-fits", "changes goes from a decision (or a clause of it) to a clause of another decision, named as 0276 §4: not %s", l)
+			}
+		case LinkSupersedes, LinkExtends:
+			if r.Kind != l.To.Kind || !isRecord || l.To.Clause > 0 {
+				refuse("link-fits", "%s goes from a record to a record of its kind: not %s", l.Kind, l)
+			}
+		}
+		if target == nil {
+			continue
+		}
+		if held != nil && held.Current.Accepted() && hasLink(held.Current, l) {
+			// An insight keeps what the accepted record linked: judged when it was accepted.
+			continue
+		}
+		switch l.Kind {
+		case LinkExtends:
+			if target.Status == StatusSuperseded && !superseders[to] {
+				refuse("rests-on", "%s extends %s, which is superseded", self, to)
+			}
+		case LinkSupersedes:
+			if !target.Accepted() || target.Status == StatusSuperseded {
+				refuse("supersedes-accepted", "%s supersedes %s, which is %s, not accepted", self, to, target.Status)
+			}
+		case LinkChanges:
+			if !target.Accepted() || target.Status == StatusSuperseded {
+				refuse("supersedes-accepted", "%s changes %s, which is %s, not accepted", self, to, target.Status)
+			} else if c := target.Clause(l.To.Clause); c != nil {
+				if (c.Status != ClauseInForce && c.Status != ClauseChanged) || c.Source == SourceDrafted {
+					refuse("changes-in-force", "%s changes %s §%d, which is %s (%s)", self, to, l.To.Clause, c.Status, c.Source)
+				}
+			}
+		case LinkCites:
+			if target.Status == StatusSuperseded {
+				named := false
+				for _, by := range supersedersOf(target) {
+					named = named || cited[by]
+				}
+				if !named {
+					refuse("live-citation", "%s cites %s, which is superseded, and not the record that replaced it", self, to)
+				}
+			}
+		}
+	}
+	if r.Kind == KindResearch && r.Effort == StatusGraduated {
+		became := false
+		for _, l := range r.Links {
+			became = became || l.Kind == LinkBecame
+		}
+		if !became {
+			refuse("became", "a graduated research effort names what it became, as a became link")
+		}
+	}
+	for _, s := range r.Sections {
+		for _, m := range mdLink.FindAllStringSubmatch(s.Text, -1) {
+			target := m[1]
+			var k Key
+			switch {
+			case researchFolder.MatchString(target):
+				n := atoi(researchFolder.FindStringSubmatch(target)[1])
+				k = Key{KindResearch, n}
+			case adrFile.MatchString(target) && !strings.Contains(target, "04-ISSUES") && !strings.Contains(target, "03-DESIGN"):
+				k = Key{KindDecision, atoi(adrFile.FindStringSubmatch(target)[1])}
+			default:
+				continue
+			}
+			if k == self {
+				continue
+			}
+			if t := lookup(k); t == nil || t.Status == StatusReserved || t.Status == StatusWithdrawn {
+				refuse("text-links", "the section %s links %s, which the knowledge base does not hold", s.Name, k)
+			}
+		}
+	}
+	text := allText(r)
+	if words := ref.RetiredIn(text); len(words) > 0 {
+		refuse("retired-word", "it uses %s, retired by the glossary", quoteAll(unique(words)))
+	}
+	if lines := ref.LinesIn(text); len(lines) > 0 {
+		refuse("daemon-line", "it writes %s, a daemon's line: the operator types nox", quoteAll(unique(lines)))
+	}
+	for _, s := range r.Sections {
+		if strings.Contains(strings.ToLower(s.Name), "option") || strings.Contains(strings.ToLower(s.Name), "rejected") {
+			continue
+		}
+		for _, p := range paragraphsOf(s.Text) {
+			if m := deferral.FindString(p); m != "" && !issueRef.MatchString(p) {
+				refuse("deferral", "the section %s puts work off (%q) and names no issue that tracks it (\"tracked by issue N\")", s.Name, m)
+			}
+		}
+	}
+	for _, c := range r.Clauses {
+		if m := deferral.FindString(c.Text); m != "" && !issueRef.MatchString(c.Text) {
+			refuse("deferral", "clause %d puts work off (%q) and names no issue that tracks it (\"tracked by issue N\")", c.N, m)
+		}
+	}
+	marked := insightMarker.FindAllStringSubmatchIndex(text, -1)
+	for _, loc := range insightPhrase.FindAllStringIndex(text, -1) {
+		inside := false
+		for _, mk := range marked {
+			inside = inside || (mk[0] <= loc[0] && loc[1] <= mk[1])
+		}
+		line := lineAround(text, loc[0])
+		if inside || strings.Contains(line, "](") || strings.HasPrefix(strings.TrimSpace(line), "#") {
+			continue
+		}
+		refuse("insight-form", "it mentions a progressive insight outside the dated marked form **Progressive insight — YYYY-MM-DD.**")
+		break
+	}
+	for _, mk := range insightMarker.FindAllStringSubmatch(text, -1) {
+		if isDay.MatchString(r.Date) && mk[1] < r.Date {
+			refuse("insight-form", "a progressive insight dated %s predates the record (%s)", mk[1], r.Date)
+		}
+	}
+	return out
+}
+
+func hasLink(r *Record, l Link) bool {
+	for _, x := range r.Links {
+		if x.Kind == l.Kind && x.From == l.From && x.To == l.To {
+			return true
+		}
+	}
+	return false
+}
+
+func lineAround(text string, at int) string {
+	start := strings.LastIndex(text[:at], "\n") + 1
+	end := strings.Index(text[at:], "\n")
+	if end < 0 {
+		return text[start:]
+	}
+	return text[start : at+end]
+}
+
+// paragraphsOf cuts a text at blank lines and a list's items.
+func paragraphsOf(text string) []string {
+	var out []string
+	for _, chunk := range strings.Split(text, "\n\n") {
+		for _, item := range regexp.MustCompile(`\n(?:[-*] |\d+\. )`).Split(chunk, -1) {
+			if strings.TrimSpace(item) != "" {
+				out = append(out, item)
+			}
+		}
+	}
+	return out
+}
+
+// supersedersOf are the records a record says superseded it: its superseded-by, a file name as hq wrote it or
+// "ADR NNNN" as an acceptance here writes it.
+func supersedersOf(r *Record) []Key {
+	var out []Key
+	for _, by := range strings.Split(r.Extra["superseded-by"], ",") {
+		by = strings.TrimSpace(by)
+		if m := adrFile.FindStringSubmatch(by); m != nil {
+			out = append(out, Key{KindDecision, atoi(m[1])})
+		} else if ref, err := ParseRef(by); err == nil {
+			if k, ok := ref.Record(); ok {
+				out = append(out, k)
+			}
+		}
+	}
+	return out
+}
+
+func atoi(s string) int {
+	n := 0
+	for _, c := range s {
+		n = n*10 + int(c-'0')
+	}
+	return n
+}
+
+func contains(list []string, s string) bool {
+	for _, x := range list {
+		if x == s {
+			return true
+		}
+	}
+	return false
+}
+
+func unique(list []string) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, s := range list {
+		if !seen[strings.ToLower(s)] {
+			seen[strings.ToLower(s)] = true
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+func quoteAll(list []string) string {
+	q := make([]string, len(list))
+	for i, s := range list {
+		q[i] = fmt.Sprintf("%q", s)
+	}
+	return strings.Join(q, ", ")
+}
+
+// ---- the whole record set ----------------------------------------------------------------------------------
+
+// WholeSetRules are the 4 rules that judge the whole record set.
+var WholeSetRules = []Rule{
+	{"cited", "a decision accepted here is cited by another record or names a home in hq", "cycle.py decisions"},
+	{"supersession", "a superseded record is named by the record that superseded it, and every record an accepted record supersedes is superseded", "records.py supersession"},
+	{"partial", "a changed clause is changed by an accepted record's changes link, and every clause an accepted record changes is changed or superseded", "records.py partial-supersession"},
+	{"rests-on-live", "no accepted record extends a superseded one, but the record that superseded it", "records.py rests-on"},
+}
+
+// JudgeSet holds the served versions of every record to the 4 whole-set rules. An imported record was held to
+// hq's own checks at the commit it was read at, so the first rule judges only a record accepted here.
+func JudgeSet(records []*Record) []Finding {
+	byKey := map[Key]*Record{}
+	for _, r := range records {
+		byKey[r.Key()] = r
+	}
+	inbound := map[Key][]struct {
+		from *Record
+		l    Link
+	}{}
+	for _, r := range records {
+		if !r.Accepted() && r.Status != StatusSuperseded {
+			continue
+		}
+		for _, l := range r.Links {
+			if to, ok := l.To.Record(); ok {
+				inbound[to] = append(inbound[to], struct {
+					from *Record
+					l    Link
+				}{r, l})
+			}
+		}
+	}
+	var out []Finding
+	add := func(rule, says string, keys ...Key) {
+		names := make([]string, len(keys))
+		for i, k := range keys {
+			names[i] = k.String()
+		}
+		out = append(out, Finding{Rule: rule, Records: names, Says: says})
+	}
+	for _, r := range records {
+		k := r.Key()
+		if r.Kind == KindDecision && r.Status == StatusAccepted && !r.AcceptedAt.IsZero() && len(r.Homes) == 0 {
+			cited := false
+			for _, in := range inbound[k] {
+				cited = cited || in.from.Key() != k
+			}
+			if !cited {
+				add("cited", fmt.Sprintf("%s is accepted and nothing cites it: no record links it and it names no home", k), k)
+			}
+		}
+		if r.Status == StatusSuperseded {
+			by := false
+			for _, in := range inbound[k] {
+				// The record that superseded it may since have been superseded itself, as hq's 0125 by 0127 by 0131.
+				by = by || in.l.Kind == LinkSupersedes
+			}
+			if !by {
+				add("supersession", fmt.Sprintf("%s is superseded and no record says it supersedes it", k), k)
+			}
+		}
+		if r.Accepted() {
+			for _, l := range r.Links {
+				to, ok := l.To.Record()
+				target := byKey[to]
+				if !ok || target == nil {
+					continue
+				}
+				switch l.Kind {
+				case LinkSupersedes:
+					if target.Status != StatusSuperseded {
+						add("supersession", fmt.Sprintf("%s supersedes %s, which is %s", k, to, target.Status), k, to)
+					}
+				case LinkChanges:
+					if l.To.Clause == 0 {
+						continue
+					}
+					if c := target.Clause(l.To.Clause); c != nil && c.Status != ClauseChanged && c.Status != ClauseSuperseded {
+						add("partial", fmt.Sprintf("%s changes %s §%d, which is %s", k, to, l.To.Clause, c.Status), k, to)
+					}
+				case LinkExtends:
+					if target.Status == StatusSuperseded {
+						superseder := false
+						for _, in := range inbound[to] {
+							superseder = superseder || (in.l.Kind == LinkSupersedes && in.from.Key() == k)
+						}
+						if !superseder {
+							add("rests-on-live", fmt.Sprintf("%s extends %s, which is superseded", k, to), k, to)
+						}
+					}
+				}
+			}
+		}
+		for _, c := range r.Clauses {
+			if c.Status != ClauseChanged {
+				continue
+			}
+			by := false
+			for _, in := range inbound[k] {
+				by = by || (in.l.Kind == LinkChanges && in.l.To.Clause == c.N && in.from.Accepted())
+			}
+			if !by {
+				add("partial", fmt.Sprintf("%s §%d is changed and no accepted record changes it", k, c.N), k)
+			}
+		}
+	}
+	return out
+}