mesh-catalog/modules/mesh-knowledge/cmd/mesh-knowledge/parse.go

package main

// Reading hq's records as they are written (novox/hq ADR 0290 rule 8, to-be 55 §8): a decision record's
// frontmatter, heading and sections, its clauses where its Decision section writes numbered statements with a
// bold lead, and its links; a research effort's overview and documents. Read the way hq's own checks read them
// (00-META/checks/records.py and cycle.py), so the import takes what those checks judged.

import (
	"fmt"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// frontmatter is the block between the first two --- lines: each key with its scalar or its list.
func frontmatter(text string) (map[string][]string, string, bool) {
	if !strings.HasPrefix(text, "---\n") {
		return nil, text, false
	}
	end := strings.Index(text[4:], "\n---")
	if end < 0 {
		return nil, text, false
	}
	block := text[4 : 4+end]
	rest := text[4+end+4:]
	rest = strings.TrimPrefix(rest, "\n")
	out := map[string][]string{}
	key := ""
	for _, line := range strings.Split(block, "\n") {
		if m := listItem.FindStringSubmatch(line); m != nil && key != "" {
			out[key] = append(out[key], strings.TrimSpace(m[1]))
			continue
		}
		m := pair.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		key = m[1]
		v := strings.TrimSpace(m[2])
		switch {
		case strings.HasPrefix(v, "[") && strings.HasSuffix(v, "]"):
			out[key] = nil
			for _, x := range strings.Split(v[1:len(v)-1], ",") {
				if x = strings.TrimSpace(x); x != "" {
					out[key] = append(out[key], x)
				}
			}
		case v == "":
			out[key] = nil
		default:
			out[key] = []string{v}
		}
	}
	return out, rest, true
}

var (
	listItem = regexp.MustCompile(`^\s+-\s+(.*)$`)
	pair     = regexp.MustCompile(`^([A-Za-z_-]+):\s*(.*)$`)
	heading  = regexp.MustCompile(`(?m)^# (\d+)\.\s*(.+)$`)
	h1       = regexp.MustCompile(`(?m)^# (.+)$`)
	// adrFile is a decision record's file name; it appears in a frontmatter value or a link target.
	adrFile = regexp.MustCompile(`(?:^|/)(\d{4})-[0-9a-z-]+\.md`)
	// researchFolder is a research effort's folder in a path.
	researchFolder = regexp.MustCompile(`01-RESEARCH/(\d{3})-[0-9a-z-]+`)
	issueFolder    = regexp.MustCompile(`04-ISSUES/(\d{3,5})-[0-9a-z-]+`)
	designPath     = regexp.MustCompile(`03-DESIGN/0[01]-[a-z-]+/[0-9a-z-]+\.md`)
	mdLink         = regexp.MustCompile(`\]\(([^)\s]+)\)`)
	// clauseStart is a numbered statement with a bold lead, at the start of a line: records.py's CLAUSE.
	clauseStart = regexp.MustCompile(`(?m)^(\d+)\. (\*\*.*)$`)
)

func first(f map[string][]string, k string) string {
	if v := f[k]; len(v) > 0 {
		return v[0]
	}
	return ""
}

// splitSections cuts a body at its `## ` headings: the text before the first (after the `# ` heading) is the
// section "preamble".
func splitSections(body string) []Section {
	var out []Section
	name := "preamble"
	var cur []string
	flush := func() {
		text := strings.Trim(strings.Join(cur, "\n"), "\n")
		if text != "" || name != "preamble" {
			out = append(out, Section{Name: name, Text: text})
		}
	}
	inFence := false
	for _, line := range strings.Split(body, "\n") {
		if strings.HasPrefix(strings.TrimSpace(line), "```") {
			inFence = !inFence
		}
		if !inFence && strings.HasPrefix(line, "## ") {
			flush()
			name, cur = strings.TrimSpace(line[3:]), nil
			continue
		}
		cur = append(cur, line)
	}
	flush()
	return out
}

// WrittenClauses are the numbered statements with a bold lead of a Decision section, each with the lines that
// continue it, in order. None when the section is prose.
func WrittenClauses(section string) []Clause {
	lines := strings.Split(section, "\n")
	var out []Clause
	var cur []string
	n := 0
	flush := func() {
		if n > 0 {
			out = append(out, Clause{N: n, Text: strings.TrimSpace(strings.Join(cur, "\n"))})
		}
	}
	for _, line := range lines {
		if m := clauseStart.FindStringSubmatch(line); m != nil {
			flush()
			n, _ = strconv.Atoi(m[1])
			cur = []string{m[2]}
			continue
		}
		if n == 0 {
			continue
		}
		// A clause goes on over its indented lines and the blank lines between them; a line at the margin that
		// is not a clause ends it.
		if strings.TrimSpace(line) == "" || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
			cur = append(cur, strings.TrimSpace(line))
			continue
		}
		flush()
		n, cur = 0, nil
	}
	flush()
	for i := range out {
		out[i].Text = collapseBlankRuns(out[i].Text)
	}
	return out
}

func collapseBlankRuns(s string) string {
	for strings.Contains(s, "\n\n\n") {
		s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
	}
	return s
}

// ParsedRecord is a record read from hq, and what the reading found odd.
type ParsedRecord struct {
	Record *Record
	Odd    []string
}

// ParseDecision reads one decision record file of hq.
func ParseDecision(file, text string) (*ParsedRecord, error) {
	m := adrFile.FindStringSubmatch(file)
	if m == nil {
		return nil, fmt.Errorf("%s is not a decision record's file (NNNN-slug.md)", file)
	}
	number, _ := strconv.Atoi(m[1])
	front, body, ok := frontmatter(text)
	if !ok {
		return nil, fmt.Errorf("%s has no frontmatter", file)
	}
	h := heading.FindStringSubmatch(body)
	if h == nil {
		return nil, fmt.Errorf("%s has no '# N. Title' heading", file)
	}
	if hn, _ := strconv.Atoi(h[1]); hn != number {
		return nil, fmt.Errorf("%s says %d in its file name and %d in its heading", file, number, hn)
	}
	p := &ParsedRecord{}
	r := &Record{Kind: KindDecision, Number: number, Title: strings.TrimSpace(h[2]), Topic: first(front, "topic"),
		Status: first(front, "status"), Date: first(front, "date"), Extra: map[string]string{}}
	for _, d := range strings.Split(first(front, "deciders"), ",") {
		if d = strings.TrimSpace(d); d != "" {
			r.Deciders = append(r.Deciders, d)
		}
	}
	after := body[strings.Index(body, h[0])+len(h[0]):]
	r.Sections = splitSections(after)
	known := map[string]bool{"topic": true, "status": true, "date": true, "deciders": true, "extends": true,
		"supersedes": true, "superseded-by": true, "supersedes-in-part": true}
	keys := make([]string, 0, len(front))
	for k := range front {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		if !known[k] {
			r.Extra[k] = strings.Join(front[k], ", ")
		}
	}
	if len(r.Extra) == 0 {
		r.Extra = nil
	}
	// Links from the frontmatter: each stored once, on the record it goes out from.
	add := func(kind string, values []string, field string) {
		for _, v := range values {
			mm := adrFile.FindStringSubmatch(v)
			if mm == nil {
				mm = regexp.MustCompile(`^(\d{4})$`).FindStringSubmatch(strings.TrimSpace(v))
			}
			if mm == nil {
				p.Odd = append(p.Odd, fmt.Sprintf("%s's %s names %q, which is not a decision record", file, field, v))
				continue
			}
			n, _ := strconv.Atoi(mm[1])
			r.Links = append(r.Links, Link{Kind: kind, To: Ref{Kind: KindDecision, Number: n}, Source: SourceWritten})
		}
	}
	add(LinkExtends, front["extends"], "extends")
	add(LinkSupersedes, front["supersedes"], "supersedes")
	add(LinkChanges, front["supersedes-in-part"], "supersedes-in-part")
	if by := first(front, "superseded-by"); by != "" {
		r.Extra = addExtra(r.Extra, "superseded-by", by)
	}
	for _, rest := range front["rests-on"] {
		add(LinkCites, []string{rest}, "rests-on")
	}
	r.Links = append(r.Links, citations(r, after)...)
	// Clauses: written, where the Decision section writes them.
	for _, s := range r.Sections {
		if strings.HasPrefix(s.Name, "Decision") {
			r.Clauses = WrittenClauses(s.Text)
			break
		}
	}
	for i := range r.Clauses {
		r.Clauses[i].Source = SourceWritten
		r.Clauses[i].Status = clauseStatusOf(r.Status)
		if r.Clauses[i].N != i+1 {
			p.Odd = append(p.Odd, fmt.Sprintf("%s numbers its clauses out of order (clause %d is the %d.)", file, r.Clauses[i].N, i+1))
		}
	}
	if len(r.Clauses) == 0 && r.Section("Decision") != "" {
		r.Drafting = &Drafting{State: DraftWaiting}
	}
	p.Record = r
	return p, nil
}

func addExtra(m map[string]string, k, v string) map[string]string {
	if m == nil {
		m = map[string]string{}
	}
	m[k] = v
	return m
}

// clauseStatusOf is a clause's status under its record's.
func clauseStatusOf(status string) string {
	switch status {
	case StatusAccepted:
		return ClauseInForce
	case StatusSuperseded:
		return ClauseSuperseded
	}
	return ClauseProposed
}

// citations are the records, issues and designs a record's text links to, once each, as `cites`; a record its
// frontmatter already links is not cited again.
func citations(r *Record, text string) []Link {
	linked := map[Ref]bool{{Kind: r.Kind, Number: r.Number}: true}
	for _, l := range r.Links {
		linked[Ref{Kind: l.To.Kind, Number: l.To.Number}] = true
	}
	var out []Link
	addRef := func(ref Ref) {
		if !linked[ref] {
			linked[ref] = true
			out = append(out, Link{Kind: LinkCites, To: ref, Source: SourceWritten})
		}
	}
	for _, m := range mdLink.FindAllStringSubmatch(text, -1) {
		target := m[1]
		switch {
		case adrFile.MatchString(target) && !strings.Contains(target, "01-RESEARCH") && !strings.Contains(target, "04-ISSUES"):
			n, _ := strconv.Atoi(adrFile.FindStringSubmatch(target)[1])
			addRef(Ref{Kind: KindDecision, Number: n})
		case researchFolder.MatchString(target):
			n, _ := strconv.Atoi(researchFolder.FindStringSubmatch(target)[1])
			addRef(Ref{Kind: KindResearch, Number: n})
		case issueFolder.MatchString(target):
			n, _ := strconv.Atoi(issueFolder.FindStringSubmatch(target)[1])
			addRef(Ref{Kind: TargetIssue, Number: n})
		case designPath.MatchString(target):
			addRef(Ref{Kind: TargetDesign, Path: designPath.FindString(target)})
		}
	}
	return out
}

// ParseResearch reads one research effort: its folder's name, its overview, and every document of it.
func ParseResearch(folder string, docs map[string]string) (*ParsedRecord, error) {
	m := regexp.MustCompile(`^(\d{3})-[0-9a-z-]+$`).FindStringSubmatch(folder)
	if m == nil {
		return nil, fmt.Errorf("%s is not a research effort's folder (NNN-slug)", folder)
	}
	number, _ := strconv.Atoi(m[1])
	overview, ok := docs["00-overview.md"]
	if !ok {
		return nil, fmt.Errorf("%s has no 00-overview.md", folder)
	}
	front, body, ok := frontmatter(overview)
	if !ok {
		return nil, fmt.Errorf("%s/00-overview.md has no frontmatter", folder)
	}
	p := &ParsedRecord{}
	r := &Record{Kind: KindResearch, Number: number, Status: first(front, "status"), Date: first(front, "initiated"),
		Topic: "research", Extra: map[string]string{}}
	if h := h1.FindStringSubmatch(body); h != nil {
		r.Title = strings.TrimSpace(h[1])
	} else {
		r.Title = folder
		p.Odd = append(p.Odd, folder+"/00-overview.md has no '# ' heading; the folder's name is its title")
	}
	for k, v := range front {
		if k != "status" && k != "initiated" && k != "became" {
			r.Extra[k] = strings.Join(v, ", ")
		}
	}
	if len(r.Extra) == 0 {
		r.Extra = nil
	}
	for _, b := range front["became"] {
		switch {
		case adrFile.MatchString(b) && strings.Contains(b, "02-DECISIONS"):
			n, _ := strconv.Atoi(adrFile.FindStringSubmatch(b)[1])
			r.Links = append(r.Links, Link{Kind: LinkBecame, To: Ref{Kind: KindDecision, Number: n}, Source: SourceWritten})
		case designPath.MatchString(b):
			r.Links = append(r.Links, Link{Kind: LinkBecame, To: Ref{Kind: TargetDesign, Path: designPath.FindString(b)}, Source: SourceWritten})
		default:
			p.Odd = append(p.Odd, fmt.Sprintf("research %03d became %q, which is neither a decision record nor a design", number, b))
		}
	}
	names := make([]string, 0, len(docs))
	for name := range docs {
		names = append(names, name)
	}
	sort.Strings(names)
	var all strings.Builder
	for _, name := range names {
		text := docs[name]
		if name == "00-overview.md" {
			text = body
		}
		r.Sections = append(r.Sections, Section{Name: name, Text: strings.Trim(text, "\n")})
		all.WriteString(text + "\n")
	}
	r.Links = append(r.Links, citations(r, all.String())...)
	p.Record = r
	return p, nil
}

// docName is a research document's name within its folder.
func docName(p string) string { return path.Base(p) }