jschoubben
1cc0ab369f
mesh/merge-gate pass: builds mesh-knowledge → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request
The import at hq f0892019 lost reconstructed, amended and a research effort's touches, held rests-on only as a citation, took eight records' clauses from a numbered list inside a ### subsection, and left the 11 records with no Decision section clause-less with nothing queued. Each is held now; refresh repairs a record only the import ever wrote, and never one changed since.
440 lines
15 KiB
Go
440 lines
15 KiB
Go
package main
|
|
|
|
// Reading hq's records as they are written (novox/hq ADR 0290 rule 8, to-be 55 §8): a decision record's
|
|
// frontmatter, heading and sections, its clauses where its Decision section writes numbered statements with a
|
|
// bold lead, and its links; a research effort's overview and documents. Read the way hq's own checks read them
|
|
// (00-META/checks/records.py and cycle.py), so the import takes what those checks judged.
|
|
|
|
import (
|
|
"fmt"
|
|
"path"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// frontmatter is the block between the first two --- lines: each key with its scalar or its list.
|
|
func frontmatter(text string) (map[string][]string, string, bool) {
|
|
if !strings.HasPrefix(text, "---\n") {
|
|
return nil, text, false
|
|
}
|
|
end := strings.Index(text[4:], "\n---")
|
|
if end < 0 {
|
|
return nil, text, false
|
|
}
|
|
block := text[4 : 4+end]
|
|
rest := text[4+end+4:]
|
|
rest = strings.TrimPrefix(rest, "\n")
|
|
out := map[string][]string{}
|
|
key := ""
|
|
for _, line := range strings.Split(block, "\n") {
|
|
if m := listItem.FindStringSubmatch(line); m != nil && key != "" {
|
|
out[key] = append(out[key], strings.TrimSpace(m[1]))
|
|
continue
|
|
}
|
|
m := pair.FindStringSubmatch(line)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
key = m[1]
|
|
v := strings.TrimSpace(m[2])
|
|
switch {
|
|
case strings.HasPrefix(v, "[") && strings.HasSuffix(v, "]"):
|
|
out[key] = nil
|
|
for _, x := range strings.Split(v[1:len(v)-1], ",") {
|
|
if x = strings.TrimSpace(x); x != "" {
|
|
out[key] = append(out[key], x)
|
|
}
|
|
}
|
|
case v == "":
|
|
out[key] = nil
|
|
default:
|
|
out[key] = []string{v}
|
|
}
|
|
}
|
|
return out, rest, true
|
|
}
|
|
|
|
var (
|
|
listItem = regexp.MustCompile(`^\s+-\s+(.*)$`)
|
|
pair = regexp.MustCompile(`^([A-Za-z_-]+):\s*(.*)$`)
|
|
heading = regexp.MustCompile(`(?m)^# (\d+)\.\s*(.+)$`)
|
|
h1 = regexp.MustCompile(`(?m)^# (.+)$`)
|
|
// adrFile is a decision record's file name; it appears in a frontmatter value or a link target.
|
|
adrFile = regexp.MustCompile(`(?:^|/)(\d{4})-[0-9a-z-]+\.md`)
|
|
// researchFolder is a research effort's folder in a path.
|
|
researchFolder = regexp.MustCompile(`01-RESEARCH/(\d{3})-[0-9a-z-]+`)
|
|
issueFolder = regexp.MustCompile(`04-ISSUES/(\d{3,5})-[0-9a-z-]+`)
|
|
designPath = regexp.MustCompile(`03-DESIGN/0[01]-[a-z-]+/[0-9a-z-]+\.md`)
|
|
mdLink = regexp.MustCompile(`\]\(([^)\s]+)\)`)
|
|
// clauseStart is a numbered statement with a bold lead, at the start of a line: records.py's CLAUSE.
|
|
clauseStart = regexp.MustCompile(`(?m)^(\d+)\. (\*\*.*)$`)
|
|
)
|
|
|
|
func first(f map[string][]string, k string) string {
|
|
if v := f[k]; len(v) > 0 {
|
|
return v[0]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// splitSections cuts a body at its `## ` headings: the text before the first (after the `# ` heading) is the
|
|
// section "preamble".
|
|
func splitSections(body string) []Section {
|
|
var out []Section
|
|
name := "preamble"
|
|
var cur []string
|
|
flush := func() {
|
|
text := strings.Trim(strings.Join(cur, "\n"), "\n")
|
|
if text != "" || name != "preamble" {
|
|
out = append(out, Section{Name: name, Text: text})
|
|
}
|
|
}
|
|
inFence := false
|
|
for _, line := range strings.Split(body, "\n") {
|
|
if strings.HasPrefix(strings.TrimSpace(line), "```") {
|
|
inFence = !inFence
|
|
}
|
|
if !inFence && strings.HasPrefix(line, "## ") {
|
|
flush()
|
|
name, cur = strings.TrimSpace(line[3:]), nil
|
|
continue
|
|
}
|
|
cur = append(cur, line)
|
|
}
|
|
flush()
|
|
return out
|
|
}
|
|
|
|
// WrittenClauses are the numbered statements with a bold lead of a Decision section's own top-level list, each
|
|
// with the lines that continue it, in order: never a list inside a `###` subsection of it, which is a part of
|
|
// that subsection's prose (ADR 0259, 0272, 0273, 0278 and 0288 number the steps of a subsection so; hq issue
|
|
// 487). None when the section is prose.
|
|
func WrittenClauses(section string) []Clause { return writtenClauses(ownText(section)) }
|
|
|
|
// subsection is a heading below the section's own level: `###` and deeper.
|
|
var subsection = regexp.MustCompile(`^#{3,}\s`)
|
|
|
|
// ownText is a section's text before its first subsection, outside a fenced block: what the section says itself.
|
|
func ownText(section string) string {
|
|
lines := strings.Split(section, "\n")
|
|
fence := false
|
|
for i, line := range lines {
|
|
if strings.HasPrefix(strings.TrimSpace(line), "```") {
|
|
fence = !fence
|
|
}
|
|
if !fence && subsection.MatchString(line) {
|
|
return strings.Join(lines[:i], "\n")
|
|
}
|
|
}
|
|
return section
|
|
}
|
|
|
|
// writtenClauses are the numbered statements with a bold lead at the margin of a text, as records.py reads them.
|
|
func writtenClauses(section string) []Clause {
|
|
lines := strings.Split(section, "\n")
|
|
var out []Clause
|
|
var cur []string
|
|
n := 0
|
|
flush := func() {
|
|
if n > 0 {
|
|
out = append(out, Clause{N: n, Text: strings.TrimSpace(strings.Join(cur, "\n"))})
|
|
}
|
|
}
|
|
for _, line := range lines {
|
|
if m := clauseStart.FindStringSubmatch(line); m != nil {
|
|
flush()
|
|
n, _ = strconv.Atoi(m[1])
|
|
cur = []string{m[2]}
|
|
continue
|
|
}
|
|
if n == 0 {
|
|
continue
|
|
}
|
|
// A clause goes on over its indented lines and the blank lines between them; a line at the margin that
|
|
// is not a clause ends it.
|
|
if strings.TrimSpace(line) == "" || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
|
|
cur = append(cur, strings.TrimSpace(line))
|
|
continue
|
|
}
|
|
flush()
|
|
n, cur = 0, nil
|
|
}
|
|
flush()
|
|
for i := range out {
|
|
out[i].Text = collapseBlankRuns(out[i].Text)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func collapseBlankRuns(s string) string {
|
|
for strings.Contains(s, "\n\n\n") {
|
|
s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
|
|
}
|
|
return s
|
|
}
|
|
|
|
// ParsedRecord is a record read from hq, and what the reading found odd.
|
|
type ParsedRecord struct {
|
|
Record *Record
|
|
Odd []string
|
|
}
|
|
|
|
// ParseDecision reads one decision record file of hq.
|
|
func ParseDecision(file, text string) (*ParsedRecord, error) {
|
|
m := adrFile.FindStringSubmatch(file)
|
|
if m == nil {
|
|
return nil, fmt.Errorf("%s is not a decision record's file (NNNN-slug.md)", file)
|
|
}
|
|
number, _ := strconv.Atoi(m[1])
|
|
front, body, ok := frontmatter(text)
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s has no frontmatter", file)
|
|
}
|
|
h := heading.FindStringSubmatch(body)
|
|
if h == nil {
|
|
return nil, fmt.Errorf("%s has no '# N. Title' heading", file)
|
|
}
|
|
if hn, _ := strconv.Atoi(h[1]); hn != number {
|
|
return nil, fmt.Errorf("%s says %d in its file name and %d in its heading", file, number, hn)
|
|
}
|
|
p := &ParsedRecord{}
|
|
r := &Record{Kind: KindDecision, Number: number, Title: strings.TrimSpace(h[2]), Topic: first(front, "topic"),
|
|
Status: first(front, "status"), Date: first(front, "date"), Extra: map[string]string{}}
|
|
for _, d := range strings.Split(first(front, "deciders"), ",") {
|
|
if d = strings.TrimSpace(d); d != "" {
|
|
r.Deciders = append(r.Deciders, d)
|
|
}
|
|
}
|
|
after := body[strings.Index(body, h[0])+len(h[0]):]
|
|
r.Sections = splitSections(after)
|
|
known := map[string]bool{"topic": true, "status": true, "date": true, "deciders": true, "extends": true,
|
|
"supersedes": true, "superseded-by": true, "supersedes-in-part": true, "reconstructed": true, "amended": true,
|
|
"rests-on": true}
|
|
keys := make([]string, 0, len(front))
|
|
for k := range front {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
if !known[k] {
|
|
r.Extra[k] = strings.Join(front[k], ", ")
|
|
}
|
|
}
|
|
if v, ok := front["reconstructed"]; ok {
|
|
switch strings.Join(v, ", ") {
|
|
case "true":
|
|
t := true
|
|
r.Reconstructed = &t
|
|
case "false":
|
|
f := false
|
|
r.Reconstructed = &f
|
|
default:
|
|
r.Extra["reconstructed"] = strings.Join(v, ", ")
|
|
p.Odd = append(p.Odd, fmt.Sprintf("%s says reconstructed %q, which is neither true nor false; it is kept as written", file, strings.Join(v, ", ")))
|
|
}
|
|
}
|
|
r.Amended = append(r.Amended, front["amended"]...)
|
|
if len(r.Extra) == 0 {
|
|
r.Extra = nil
|
|
}
|
|
// Links from the frontmatter: each stored once, on the record it goes out from.
|
|
add := func(kind string, values []string, field string) {
|
|
for _, v := range values {
|
|
mm := adrFile.FindStringSubmatch(v)
|
|
if mm == nil {
|
|
mm = regexp.MustCompile(`^(\d{4})$`).FindStringSubmatch(strings.TrimSpace(v))
|
|
}
|
|
if mm == nil {
|
|
p.Odd = append(p.Odd, fmt.Sprintf("%s's %s names %q, which is not a decision record", file, field, v))
|
|
continue
|
|
}
|
|
n, _ := strconv.Atoi(mm[1])
|
|
r.Links = append(r.Links, Link{Kind: kind, To: Ref{Kind: KindDecision, Number: n}, Source: SourceWritten})
|
|
}
|
|
}
|
|
add(LinkExtends, front["extends"], "extends")
|
|
add(LinkSupersedes, front["supersedes"], "supersedes")
|
|
add(LinkChanges, front["supersedes-in-part"], "supersedes-in-part")
|
|
if by := first(front, "superseded-by"); by != "" {
|
|
r.Extra = addExtra(r.Extra, "superseded-by", by)
|
|
}
|
|
add(LinkRestsOn, front["rests-on"], "rests-on")
|
|
r.Links = append(r.Links, citations(r, after)...)
|
|
// Clauses: written, where the Decision section's own list writes them.
|
|
section, hasDecision := decisionSection(r)
|
|
if hasDecision {
|
|
r.Clauses = WrittenClauses(section)
|
|
}
|
|
for i := range r.Clauses {
|
|
r.Clauses[i].Source = SourceWritten
|
|
r.Clauses[i].Status = clauseStatusOf(r.Status)
|
|
if r.Clauses[i].N != i+1 {
|
|
p.Odd = append(p.Odd, fmt.Sprintf("%s numbers its clauses out of order (clause %d is the %d.)", file, r.Clauses[i].N, i+1))
|
|
}
|
|
}
|
|
// A record without written clauses is drafted: from its Decision section, or, where it has none (11 of hq's
|
|
// records), from what it decides across its sections. Never clause-less in silence.
|
|
switch {
|
|
case len(r.Clauses) > 0:
|
|
case hasDecision && strings.TrimSpace(section) != "":
|
|
r.Drafting = &Drafting{State: DraftWaiting}
|
|
default:
|
|
r.Drafting = &Drafting{State: DraftWaiting, Reason: NoDecisionSection}
|
|
}
|
|
p.Record = r
|
|
return p, nil
|
|
}
|
|
|
|
func addExtra(m map[string]string, k, v string) map[string]string {
|
|
if m == nil {
|
|
m = map[string]string{}
|
|
}
|
|
m[k] = v
|
|
return m
|
|
}
|
|
|
|
// clauseStatusOf is a clause's status under its record's.
|
|
func clauseStatusOf(status string) string {
|
|
switch status {
|
|
case StatusAccepted:
|
|
return ClauseInForce
|
|
case StatusSuperseded:
|
|
return ClauseSuperseded
|
|
}
|
|
return ClauseProposed
|
|
}
|
|
|
|
// citations are the records, issues and designs a record's text links to, once each, as `cites`; a record its
|
|
// frontmatter already links is not cited again.
|
|
func citations(r *Record, text string) []Link {
|
|
linked := map[Ref]bool{{Kind: r.Kind, Number: r.Number}: true}
|
|
for _, l := range r.Links {
|
|
linked[Ref{Kind: l.To.Kind, Number: l.To.Number}] = true
|
|
}
|
|
var out []Link
|
|
addRef := func(ref Ref) {
|
|
if !linked[ref] {
|
|
linked[ref] = true
|
|
out = append(out, Link{Kind: LinkCites, To: ref, Source: SourceWritten})
|
|
}
|
|
}
|
|
for _, m := range mdLink.FindAllStringSubmatch(text, -1) {
|
|
target := m[1]
|
|
switch {
|
|
case adrFile.MatchString(target) && !strings.Contains(target, "01-RESEARCH") && !strings.Contains(target, "04-ISSUES"):
|
|
n, _ := strconv.Atoi(adrFile.FindStringSubmatch(target)[1])
|
|
addRef(Ref{Kind: KindDecision, Number: n})
|
|
case researchFolder.MatchString(target):
|
|
n, _ := strconv.Atoi(researchFolder.FindStringSubmatch(target)[1])
|
|
addRef(Ref{Kind: KindResearch, Number: n})
|
|
case issueFolder.MatchString(target):
|
|
n, _ := strconv.Atoi(issueFolder.FindStringSubmatch(target)[1])
|
|
addRef(Ref{Kind: TargetIssue, Number: n})
|
|
case designPath.MatchString(target):
|
|
addRef(Ref{Kind: TargetDesign, Path: designPath.FindString(target)})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ParseResearch reads one research effort: its folder's name, its overview, and every document of it.
|
|
func ParseResearch(folder string, docs map[string]string) (*ParsedRecord, error) {
|
|
m := regexp.MustCompile(`^(\d{3})-[0-9a-z-]+$`).FindStringSubmatch(folder)
|
|
if m == nil {
|
|
return nil, fmt.Errorf("%s is not a research effort's folder (NNN-slug)", folder)
|
|
}
|
|
number, _ := strconv.Atoi(m[1])
|
|
overview, ok := docs["00-overview.md"]
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s has no 00-overview.md", folder)
|
|
}
|
|
front, body, ok := frontmatter(overview)
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s/00-overview.md has no frontmatter", folder)
|
|
}
|
|
p := &ParsedRecord{}
|
|
r := &Record{Kind: KindResearch, Number: number, Status: first(front, "status"), Date: first(front, "initiated"),
|
|
Topic: "research", Extra: map[string]string{}}
|
|
if h := h1.FindStringSubmatch(body); h != nil {
|
|
r.Title = strings.TrimSpace(h[1])
|
|
} else {
|
|
r.Title = folder
|
|
p.Odd = append(p.Odd, folder+"/00-overview.md has no '# ' heading; the folder's name is its title")
|
|
}
|
|
for k, v := range front {
|
|
if k != "status" && k != "initiated" && k != "became" && k != "touches" {
|
|
r.Extra[k] = strings.Join(v, ", ")
|
|
}
|
|
}
|
|
// What it touches: a link where it names a record, an issue or a design; as written otherwise.
|
|
touched := map[Ref]bool{}
|
|
for _, v := range front["touches"] {
|
|
if ref, ok := refOfPath(v); ok && !(ref.Kind == KindResearch && ref.Number == number) {
|
|
if !touched[ref] {
|
|
touched[ref] = true
|
|
r.Links = append(r.Links, Link{Kind: LinkTouches, To: ref, Source: SourceWritten})
|
|
}
|
|
continue
|
|
}
|
|
r.Touches = append(r.Touches, v)
|
|
}
|
|
if len(r.Extra) == 0 {
|
|
r.Extra = nil
|
|
}
|
|
for _, b := range front["became"] {
|
|
switch {
|
|
case adrFile.MatchString(b) && strings.Contains(b, "02-DECISIONS"):
|
|
n, _ := strconv.Atoi(adrFile.FindStringSubmatch(b)[1])
|
|
r.Links = append(r.Links, Link{Kind: LinkBecame, To: Ref{Kind: KindDecision, Number: n}, Source: SourceWritten})
|
|
case designPath.MatchString(b):
|
|
r.Links = append(r.Links, Link{Kind: LinkBecame, To: Ref{Kind: TargetDesign, Path: designPath.FindString(b)}, Source: SourceWritten})
|
|
default:
|
|
p.Odd = append(p.Odd, fmt.Sprintf("research %03d became %q, which is neither a decision record nor a design", number, b))
|
|
}
|
|
}
|
|
names := make([]string, 0, len(docs))
|
|
for name := range docs {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
var all strings.Builder
|
|
for _, name := range names {
|
|
text := docs[name]
|
|
if name == "00-overview.md" {
|
|
text = body
|
|
}
|
|
r.Sections = append(r.Sections, Section{Name: name, Text: strings.Trim(text, "\n")})
|
|
all.WriteString(text + "\n")
|
|
}
|
|
r.Links = append(r.Links, citations(r, all.String())...)
|
|
p.Record = r
|
|
return p, nil
|
|
}
|
|
|
|
// NoDecisionSection is the drafting reason of a decision record that has no Decision section.
|
|
const NoDecisionSection = "no Decision section: its clauses are drafted from what it decides across its sections"
|
|
|
|
// refOfPath is the record, issue or design a path of hq names, if it names one.
|
|
func refOfPath(v string) (Ref, bool) {
|
|
v = strings.TrimSpace(v)
|
|
switch {
|
|
case issueFolder.MatchString(v):
|
|
n, _ := strconv.Atoi(issueFolder.FindStringSubmatch(v)[1])
|
|
return Ref{Kind: TargetIssue, Number: n}, true
|
|
case researchFolder.MatchString(v):
|
|
n, _ := strconv.Atoi(researchFolder.FindStringSubmatch(v)[1])
|
|
return Ref{Kind: KindResearch, Number: n}, true
|
|
case adrFile.MatchString(v) && (strings.Contains(v, DecisionsFolder) || !strings.Contains(v, "/")):
|
|
n, _ := strconv.Atoi(adrFile.FindStringSubmatch(v)[1])
|
|
return Ref{Kind: KindDecision, Number: n}, true
|
|
case designPath.MatchString(v):
|
|
return Ref{Kind: TargetDesign, Path: designPath.FindString(v)}, true
|
|
}
|
|
return Ref{}, false
|
|
}
|
|
|
|
// docName is a research document's name within its folder.
|
|
func docName(p string) string { return path.Base(p) }
|