mesh-knowledge: the import keeps every field and takes clauses only from the Decision section's own list, and can repair what it held (hq issue 487)

The import at hq f0892019 lost reconstructed, amended and a research effort's touches, held rests-on only as a
citation, took eight records' clauses from a numbered list inside a ### subsection, and left the 11 records with
no Decision section clause-less with nothing queued. Each is held now; refresh repairs a record only the import
ever wrote, and never one changed since.

modules/mesh-knowledge/cmd/mesh-knowledge/import.go

@@ -9,6 +9,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"regexp"
@@ -29,10 +30,13 @@ const LinkPending = "pending"
 
 var commitPrefix = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
 
-// ImportAsk is what an import takes.
+// ImportAsk is what an import takes. Refresh also repairs a record held whose only versions are the imports' own
+// (hq issue 487): it is read again at the commit and written as a new version naming what was lost and why. A
+// record anyone changed after its import is never touched.
 type ImportAsk struct {
-	Commit string
-	DryRun bool
+	Commit  string
+	DryRun  bool
+	Refresh bool
 }
 
 // ImportRow is what the import did, or would do, with one record.
@@ -50,24 +54,33 @@ const (
 	ImportHeld    = "held"
 	ImportRefused = "refused"
 	ImportWould   = "would-create"
+	// A refresh's: a record repaired, or one that would be; and one changed here since its import, left alone.
+	ImportRefresh      = "refresh"
+	ImportWouldRefresh = "would-refresh"
+	ImportChangedSince = "changed-since"
 )
 
 // Imported is the answer to an import.
 type Imported struct {
-	DryRun     bool        `json:"dry-run"`
-	Commit     string      `json:"commit"`
-	Read       int         `json:"read"`
-	Created    int         `json:"created"`
-	Held       int         `json:"held"`
-	Refused    int         `json:"refused"`
-	Clauses    int         `json:"written-clauses"`
-	ToDraft    int         `json:"prose-decisions"`
-	Rows       []ImportRow `json:"rows"`
-	Odd        []string    `json:"odd,omitempty"`
-	Cut        string      `json:"cut,omitempty"`
-	Said       string      `json:"said"`
-	Findings   []Finding   `json:"whole-set-findings,omitempty"`
-	Unresolved int         `json:"unresolved-links"`
+	DryRun  bool   `json:"dry-run"`
+	Commit  string `json:"commit"`
+	Read    int    `json:"read"`
+	Created int    `json:"created"`
+	Held    int    `json:"held"`
+	Refused int    `json:"refused"`
+	// A refresh's counts: repaired (or would be), read again and found the same, and changed here since their
+	// import, which a refresh never touches.
+	Refreshed    int         `json:"refreshed,omitempty"`
+	Unchanged    int         `json:"unchanged,omitempty"`
+	ChangedSince int         `json:"changed-since,omitempty"`
+	Clauses      int         `json:"written-clauses"`
+	ToDraft      int         `json:"prose-decisions"`
+	Rows         []ImportRow `json:"rows"`
+	Odd          []string    `json:"odd,omitempty"`
+	Cut          string      `json:"cut,omitempty"`
+	Said         string      `json:"said"`
+	Findings     []Finding   `json:"whole-set-findings,omitempty"`
+	Unresolved   int         `json:"unresolved-links"`
 }
 
 // MostImportRows is how many rows an import answers; the counts are always whole.
@@ -199,9 +212,9 @@ func (k *KB) Import(caller string, a ImportAsk, records Records) (*Imported, err
 	if err != nil {
 		return nil, fmt.Errorf("the knowledge base cannot be read (%v); nothing was imported", err)
 	}
-	held := map[Key]bool{}
+	held := map[Key]*Entry{}
 	for _, e := range all {
-		held[e.Current.Key()] = true
+		held[e.Current.Key()] = e
 		known[e.Current.Key()] = true
 	}
 	at := k.now()
@@ -219,22 +232,31 @@ func (k *KB) Import(caller string, a ImportAsk, records Records) (*Imported, err
 		row.Record, row.Status = r.Key().String(), r.Status
 		out.Read++
 		out.Odd = append(out.Odd, c.parsed.Odd...)
-		if held[r.Key()] {
-			row.Action = ImportHeld
-			out.Held++
-			continue
-		}
 		for i, l := range r.Links {
 			if to, ok := l.To.Record(); ok && !known[to] {
 				r.Links[i].Unresolved = true
-				out.Unresolved++
-				out.Odd = append(out.Odd, fmt.Sprintf("%s %s %s, which hq does not hold at this commit", r.Key(), l.Kind, to))
+				if held[r.Key()] == nil {
+					out.Unresolved++
+					out.Odd = append(out.Odd, fmt.Sprintf("%s %s %s, which hq does not hold at this commit", r.Key(), l.Kind, to))
+				}
 			}
 			if l.Kind == LinkChanges && l.To.Clause == 0 {
 				r.Links[i].Source = LinkPending
 			}
 		}
 		r.Source = "hq@" + commit + ":" + c.path
+		if e := held[r.Key()]; e != nil {
+			if !a.Refresh {
+				row.Action = ImportHeld
+				out.Held++
+				continue
+			}
+			k.refresh(ctx, a, e, r, c.path, commit, caller, &row, out)
+			if row.Action != ImportHeld {
+				rows = append(rows, row)
+			}
+			continue
+		}
 		out.Clauses += len(r.Clauses)
 		if r.Drafting != nil {
 			out.ToDraft++
@@ -267,23 +289,194 @@ func (k *KB) Import(caller string, a ImportAsk, records Records) (*Imported, err
 	if len(out.Odd) > 40 {
 		out.Odd = append(out.Odd[:40], fmt.Sprintf("and %d more", len(out.Odd)-40))
 	}
-	verb := "created"
+	verb, again := "created", "refreshed"
 	if a.DryRun {
-		verb = "would create"
+		verb, again = "would create", "would refresh"
 	}
 	out.Said = fmt.Sprintf("Read %d records of hq at commit %s: %s %d, already held %d, refused %d; %d clauses written as such, "+
 		"%d decisions written as prose wait for their clauses to be drafted and confirmed.",
-		out.Read, short(commit), verb, out.Created, out.Held, out.Refused, out.Clauses, out.ToDraft)
+		out.Read, short(commit), verb, out.Created, out.Held+out.Refreshed+out.Unchanged+out.ChangedSince, out.Refused, out.Clauses, out.ToDraft)
+	if a.Refresh {
+		out.Said += fmt.Sprintf(" Of those held: %s %d that only the import ever wrote, %d read again and the same, %d changed here "+
+			"since their import and left as they are.", again, out.Refreshed, out.Unchanged, out.ChangedSince)
+	}
 	switch {
 	case a.DryRun:
 		out.Said += " Nothing was written: a dry run."
-	case out.Created == 0 && out.Refused == 0:
-		out.Said += " Nothing changed: every record is already held."
-	case out.Created == 0:
+	case out.Created == 0 && out.Refreshed == 0 && out.Refused == 0:
+		out.Said += " Nothing changed: every record is already held as hq says it."
+	case out.Created == 0 && out.Refreshed == 0:
 		out.Said += " Nothing changed: every record not refused is already held."
 	}
-	if !a.DryRun && out.Created > 0 {
+	if !a.DryRun && out.Created+out.Refreshed > 0 {
 		out.Findings = k.judgeSet(ctx)
 	}
 	return out, nil
 }
+
+// importedOnly says whether every version of a record was written by the import or a refresh of it: nobody has
+// changed it since it was read from hq.
+func importedOnly(vs []Version) bool {
+	if len(vs) == 0 {
+		return false
+	}
+	for _, v := range vs {
+		if v.Insight || v.Record == nil || !strings.HasPrefix(v.Record.Source, "hq@") ||
+			!(strings.HasPrefix(v.Why, "imported from hq at ") || strings.HasPrefix(v.Why, "refreshed from hq at ")) {
+			return false
+		}
+	}
+	return true
+}
+
+// refresh reads a held record again (r, as hq has it at the commit) and, when only the import ever wrote it and
+// what it holds differs, writes r as its new version naming what changed and why. A record changed here since
+// its import is never touched; a second refresh finds the same and changes nothing.
+func (k *KB) refresh(ctx context.Context, a ImportAsk, e *Entry, r *Record, path, commit, caller string, row *ImportRow, out *Imported) {
+	vs, err := k.Store.Versions(ctx, r.Key())
+	if err != nil {
+		row.Action, row.Why = ImportRefused, "its versions cannot be read: "+err.Error()
+		out.Refused++
+		return
+	}
+	if e.Pending != nil || !importedOnly(vs) {
+		last := vs[len(vs)-1]
+		row.Action = ImportChangedSince
+		row.Why = fmt.Sprintf("changed here since its import (version %d by %s: %s); a refresh never touches it", last.N, last.By, last.Why)
+		if e.Pending != nil {
+			row.Why = "a version waits for its review over it; a refresh never touches it"
+		}
+		out.ChangedSince++
+		return
+	}
+	diffs := differences(e.Current, r)
+	if len(diffs) == 0 {
+		row.Action = ImportHeld
+		out.Unchanged++
+		return
+	}
+	row.Why = strings.Join(diffs, "; ")
+	row.Action = ImportWouldRefresh
+	out.Refreshed++
+	if a.DryRun {
+		return
+	}
+	why := fmt.Sprintf("refreshed from hq at %s: %s; the import before hq issue 487 had lost or misread %s", short(commit), path, row.Why)
+	if len(why) > 1000 {
+		why = why[:997] + "..."
+	}
+	if err := k.Store.Save(ctx, Version{By: caller, At: k.now(), Why: why, Record: r}, true, false, e.Revision); err != nil {
+		row.Action, row.Why = ImportRefused, "it could not be written: "+err.Error()
+		out.Refused++
+		out.Refreshed--
+		return
+	}
+	row.Action = ImportRefresh
+	k.emit(EventChanged, r, caller, why, e.Versions+1)
+}
+
+// differences names what a record read again from hq holds that the one held does not, field by field; none when
+// the two hold the same.
+func differences(old, now *Record) []string {
+	var out []string
+	o, n := old.Clone(), now.Clone()
+	o.Source, n.Source = "", ""
+	if a, b := renderJSON(o), renderJSON(n); a == b {
+		return nil
+	}
+	field := func(name, a, b string) {
+		if a != b {
+			if a == "" {
+				a = "unset"
+			}
+			if b == "" {
+				b = "unset"
+			}
+			out = append(out, fmt.Sprintf("%s: %s → %s", name, a, b))
+		}
+	}
+	boolOf := func(p *bool) string {
+		if p == nil {
+			return ""
+		}
+		return fmt.Sprint(*p)
+	}
+	field("title", o.Title, n.Title)
+	field("status", o.Status, n.Status)
+	field("topic", o.Topic, n.Topic)
+	field("date", o.Date, n.Date)
+	field("deciders", strings.Join(o.Deciders, ", "), strings.Join(n.Deciders, ", "))
+	field("reconstructed", boolOf(o.Reconstructed), boolOf(n.Reconstructed))
+	field("amended", strings.Join(o.Amended, ", "), strings.Join(n.Amended, ", "))
+	field("touches", strings.Join(o.Touches, ", "), strings.Join(n.Touches, ", "))
+	var gone []string
+	for key := range o.Extra {
+		if _, ok := n.Extra[key]; !ok {
+			gone = append(gone, key)
+		}
+	}
+	sort.Strings(gone)
+	if len(gone) > 0 {
+		out = append(out, "held as a field of its own now: "+strings.Join(gone, ", "))
+	}
+	if renderJSON(o.Clauses) != renderJSON(n.Clauses) {
+		s := fmt.Sprintf("clauses: %d → %d", len(o.Clauses), len(n.Clauses))
+		if len(n.Clauses) == 0 && n.Drafting != nil {
+			s += ", to be drafted"
+		}
+		out = append(out, s)
+	}
+	if (o.Drafting == nil) != (n.Drafting == nil) || (n.Drafting != nil && o.Drafting != nil && o.Drafting.Reason != n.Drafting.Reason) {
+		if n.Drafting != nil {
+			s := "drafting queued"
+			if n.Drafting.Reason != "" {
+				s += " (" + n.Drafting.Reason + ")"
+			}
+			out = append(out, s)
+		} else {
+			out = append(out, "no drafting: its clauses are written")
+		}
+	}
+	ol, nl := map[string]bool{}, map[string]bool{}
+	for _, l := range o.Links {
+		ol[l.String()] = true
+	}
+	for _, l := range n.Links {
+		nl[l.String()] = true
+	}
+	var added, removed []string
+	for l := range nl {
+		if !ol[l] {
+			added = append(added, l)
+		}
+	}
+	for l := range ol {
+		if !nl[l] {
+			removed = append(removed, l)
+		}
+	}
+	sort.Strings(added)
+	sort.Strings(removed)
+	if len(added)+len(removed) > 0 {
+		s := "links:"
+		if len(added) > 0 {
+			s += " + " + strings.Join(added, ", + ")
+		}
+		if len(removed) > 0 {
+			s += " − " + strings.Join(removed, ", − ")
+		}
+		out = append(out, s)
+	}
+	if renderJSON(o.Sections) != renderJSON(n.Sections) {
+		out = append(out, "its sections")
+	}
+	if len(out) == 0 {
+		out = append(out, "what it holds otherwise (as hq writes it)")
+	}
+	return out
+}
+
+func renderJSON(v any) string {
+	raw, _ := json.Marshal(v)
+	return string(raw)
+}