mesh-knowledge: the import keeps every field and takes clauses only from the Decision section's own list, and can repair what it held (hq issue 487)

The import at hq f0892019 lost reconstructed, amended and a research effort's touches, held rests-on only as a
citation, took eight records' clauses from a numbered list inside a ### subsection, and left the 11 records with
no Decision section clause-less with nothing queued. Each is held now; refresh repairs a record only the import
ever wrote, and never one changed since.

modules/mesh-knowledge/cmd/mesh-knowledge/parse.go

@@ -107,9 +107,32 @@ func splitSections(body string) []Section {
 	return out
 }
 
-// WrittenClauses are the numbered statements with a bold lead of a Decision section, each with the lines that
-// continue it, in order. None when the section is prose.
-func WrittenClauses(section string) []Clause {
+// WrittenClauses are the numbered statements with a bold lead of a Decision section's own top-level list, each
+// with the lines that continue it, in order: never a list inside a `###` subsection of it, which is a part of
+// that subsection's prose (ADR 0259, 0272, 0273, 0278 and 0288 number the steps of a subsection so; hq issue
+// 487). None when the section is prose.
+func WrittenClauses(section string) []Clause { return writtenClauses(ownText(section)) }
+
+// subsection is a heading below the section's own level: `###` and deeper.
+var subsection = regexp.MustCompile(`^#{3,}\s`)
+
+// ownText is a section's text before its first subsection, outside a fenced block: what the section says itself.
+func ownText(section string) string {
+	lines := strings.Split(section, "\n")
+	fence := false
+	for i, line := range lines {
+		if strings.HasPrefix(strings.TrimSpace(line), "```") {
+			fence = !fence
+		}
+		if !fence && subsection.MatchString(line) {
+			return strings.Join(lines[:i], "\n")
+		}
+	}
+	return section
+}
+
+// writtenClauses are the numbered statements with a bold lead at the margin of a text, as records.py reads them.
+func writtenClauses(section string) []Clause {
 	lines := strings.Split(section, "\n")
 	var out []Clause
 	var cur []string
@@ -187,7 +210,8 @@ func ParseDecision(file, text string) (*ParsedRecord, error) {
 	after := body[strings.Index(body, h[0])+len(h[0]):]
 	r.Sections = splitSections(after)
 	known := map[string]bool{"topic": true, "status": true, "date": true, "deciders": true, "extends": true,
-		"supersedes": true, "superseded-by": true, "supersedes-in-part": true}
+		"supersedes": true, "superseded-by": true, "supersedes-in-part": true, "reconstructed": true, "amended": true,
+		"rests-on": true}
 	keys := make([]string, 0, len(front))
 	for k := range front {
 		keys = append(keys, k)
@@ -198,6 +222,20 @@ func ParseDecision(file, text string) (*ParsedRecord, error) {
 			r.Extra[k] = strings.Join(front[k], ", ")
 		}
 	}
+	if v, ok := front["reconstructed"]; ok {
+		switch strings.Join(v, ", ") {
+		case "true":
+			t := true
+			r.Reconstructed = &t
+		case "false":
+			f := false
+			r.Reconstructed = &f
+		default:
+			r.Extra["reconstructed"] = strings.Join(v, ", ")
+			p.Odd = append(p.Odd, fmt.Sprintf("%s says reconstructed %q, which is neither true nor false; it is kept as written", file, strings.Join(v, ", ")))
+		}
+	}
+	r.Amended = append(r.Amended, front["amended"]...)
 	if len(r.Extra) == 0 {
 		r.Extra = nil
 	}
@@ -222,16 +260,12 @@ func ParseDecision(file, text string) (*ParsedRecord, error) {
 	if by := first(front, "superseded-by"); by != "" {
 		r.Extra = addExtra(r.Extra, "superseded-by", by)
 	}
-	for _, rest := range front["rests-on"] {
-		add(LinkCites, []string{rest}, "rests-on")
-	}
+	add(LinkRestsOn, front["rests-on"], "rests-on")
 	r.Links = append(r.Links, citations(r, after)...)
-	// Clauses: written, where the Decision section writes them.
-	for _, s := range r.Sections {
-		if strings.HasPrefix(s.Name, "Decision") {
-			r.Clauses = WrittenClauses(s.Text)
-			break
-		}
+	// Clauses: written, where the Decision section's own list writes them.
+	section, hasDecision := decisionSection(r)
+	if hasDecision {
+		r.Clauses = WrittenClauses(section)
 	}
 	for i := range r.Clauses {
 		r.Clauses[i].Source = SourceWritten
@@ -240,8 +274,14 @@ func ParseDecision(file, text string) (*ParsedRecord, error) {
 			p.Odd = append(p.Odd, fmt.Sprintf("%s numbers its clauses out of order (clause %d is the %d.)", file, r.Clauses[i].N, i+1))
 		}
 	}
-	if len(r.Clauses) == 0 && r.Section("Decision") != "" {
+	// A record without written clauses is drafted: from its Decision section, or, where it has none (11 of hq's
+	// records), from what it decides across its sections. Never clause-less in silence.
+	switch {
+	case len(r.Clauses) > 0:
+	case hasDecision && strings.TrimSpace(section) != "":
 		r.Drafting = &Drafting{State: DraftWaiting}
+	default:
+		r.Drafting = &Drafting{State: DraftWaiting, Reason: NoDecisionSection}
 	}
 	p.Record = r
 	return p, nil
@@ -324,10 +364,22 @@ func ParseResearch(folder string, docs map[string]string) (*ParsedRecord, error)
 		p.Odd = append(p.Odd, folder+"/00-overview.md has no '# ' heading; the folder's name is its title")
 	}
 	for k, v := range front {
-		if k != "status" && k != "initiated" && k != "became" {
+		if k != "status" && k != "initiated" && k != "became" && k != "touches" {
 			r.Extra[k] = strings.Join(v, ", ")
 		}
 	}
+	// What it touches: a link where it names a record, an issue or a design; as written otherwise.
+	touched := map[Ref]bool{}
+	for _, v := range front["touches"] {
+		if ref, ok := refOfPath(v); ok && !(ref.Kind == KindResearch && ref.Number == number) {
+			if !touched[ref] {
+				touched[ref] = true
+				r.Links = append(r.Links, Link{Kind: LinkTouches, To: ref, Source: SourceWritten})
+			}
+			continue
+		}
+		r.Touches = append(r.Touches, v)
+	}
 	if len(r.Extra) == 0 {
 		r.Extra = nil
 	}
@@ -361,5 +413,27 @@ func ParseResearch(folder string, docs map[string]string) (*ParsedRecord, error)
 	return p, nil
 }
 
+// NoDecisionSection is the drafting reason of a decision record that has no Decision section.
+const NoDecisionSection = "no Decision section: its clauses are drafted from what it decides across its sections"
+
+// refOfPath is the record, issue or design a path of hq names, if it names one.
+func refOfPath(v string) (Ref, bool) {
+	v = strings.TrimSpace(v)
+	switch {
+	case issueFolder.MatchString(v):
+		n, _ := strconv.Atoi(issueFolder.FindStringSubmatch(v)[1])
+		return Ref{Kind: TargetIssue, Number: n}, true
+	case researchFolder.MatchString(v):
+		n, _ := strconv.Atoi(researchFolder.FindStringSubmatch(v)[1])
+		return Ref{Kind: KindResearch, Number: n}, true
+	case adrFile.MatchString(v) && (strings.Contains(v, DecisionsFolder) || !strings.Contains(v, "/")):
+		n, _ := strconv.Atoi(adrFile.FindStringSubmatch(v)[1])
+		return Ref{Kind: KindDecision, Number: n}, true
+	case designPath.MatchString(v):
+		return Ref{Kind: TargetDesign, Path: designPath.FindString(v)}, true
+	}
+	return Ref{}, false
+}
+
 // docName is a research document's name within its folder.
 func docName(p string) string { return path.Base(p) }