548 lines
18 KiB
Go
548 lines
18 KiB
Go
package main
|
||
|
||
// The import (novox/hq ADR 0290 rule 8, to-be 55 §8): every decision record and research effort of hq, read at
|
||
// one commit through the records module, written whole at its own number as its first version, naming the commit
|
||
// and path it came from. It creates what is absent and never overwrites; it never asks the issue tracker for a
|
||
// number, so no counter moves; a second run changes nothing and says so. A dry run reads everything, writes
|
||
// nothing, and says what it would do. A record whose Decision section is prose is written with no clauses and
|
||
// waits for the drafting and confirming of its clauses (draft.go).
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
)
|
||
|
||
// The folders of hq the import reads.
|
||
const (
|
||
DecisionsFolder = "02-DECISIONS"
|
||
ResearchFolder = "01-RESEARCH"
|
||
)
|
||
|
||
// LinkPending is the source of a `changes` link the import read from hq's supersedes-in-part, which named the
|
||
// record and not the clause: which clause it changes is drafted and confirmed by the judgement (to-be 55 §7).
|
||
const LinkPending = "pending"
|
||
|
||
var commitPrefix = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||
|
||
// ImportAsk is what an import takes. Refresh also repairs a record held whose only versions are the imports' own
|
||
// (hq issue 487): it is read again at the commit and written as a new version naming what was lost and why. A
|
||
// record anyone changed after its import is never touched.
|
||
type ImportAsk struct {
|
||
Commit string
|
||
DryRun bool
|
||
Refresh bool
|
||
}
|
||
|
||
// ImportRow is what the import did, or would do, with one record.
|
||
type ImportRow struct {
|
||
Record string `json:"record"`
|
||
Action string `json:"action"`
|
||
Status string `json:"status,omitempty"`
|
||
From string `json:"from"`
|
||
Why string `json:"why,omitempty"`
|
||
}
|
||
|
||
// The actions of an import.
|
||
const (
|
||
ImportCreate = "create"
|
||
ImportHeld = "held"
|
||
ImportRefused = "refused"
|
||
ImportWould = "would-create"
|
||
// A refresh's: a record repaired, or one that would be; and one changed here since its import, left alone.
|
||
ImportRefresh = "refresh"
|
||
ImportWouldRefresh = "would-refresh"
|
||
ImportChangedSince = "changed-since"
|
||
)
|
||
|
||
// Imported is the answer to an import.
|
||
type Imported struct {
|
||
DryRun bool `json:"dry-run"`
|
||
Commit string `json:"commit"`
|
||
Read int `json:"read"`
|
||
Created int `json:"created"`
|
||
Held int `json:"held"`
|
||
Refused int `json:"refused"`
|
||
// A refresh's counts: repaired (or would be), read again and found the same, and changed here since their
|
||
// import, which a refresh never touches.
|
||
Refreshed int `json:"refreshed,omitempty"`
|
||
Unchanged int `json:"unchanged,omitempty"`
|
||
ChangedSince int `json:"changed-since,omitempty"`
|
||
Clauses int `json:"written-clauses"`
|
||
ToDraft int `json:"prose-decisions"`
|
||
Rows []ImportRow `json:"rows"`
|
||
Odd []string `json:"odd,omitempty"`
|
||
Cut string `json:"cut,omitempty"`
|
||
Said string `json:"said"`
|
||
Findings []Finding `json:"whole-set-findings,omitempty"`
|
||
Unresolved int `json:"unresolved-links"`
|
||
}
|
||
|
||
// MostImportRows is how many rows an import answers; the counts are always whole.
|
||
const MostImportRows = 60
|
||
|
||
const importReaders = 8
|
||
|
||
type importCandidate struct {
|
||
path string
|
||
parsed *ParsedRecord
|
||
err error
|
||
}
|
||
|
||
// Import reads hq at its commit and writes every record it does not hold.
|
||
func (k *KB) Import(caller string, a ImportAsk, records Records) (*Imported, error) {
|
||
commit, err := records.Standing()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("hq cannot be read through the records module (%v); nothing was imported", err)
|
||
}
|
||
want := strings.TrimSpace(a.Commit)
|
||
if want != "" && !commitPrefix.MatchString(want) {
|
||
return nil, errors.New("commit is a commit's id or a prefix of it, 7 to 40 hexadecimal digits; nothing was imported")
|
||
}
|
||
if want != "" && !strings.HasPrefix(commit, want) {
|
||
return nil, fmt.Errorf("the record stands at hq commit %s, not %s: bring it there (records_sync), or name the commit it "+
|
||
"stands at. Nothing was imported", short(commit), want)
|
||
}
|
||
_, docs, err := records.List(DecisionsFolder)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("hq's %s cannot be listed (%v); nothing was imported", DecisionsFolder, err)
|
||
}
|
||
efforts, _, err := records.List(ResearchFolder)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("hq's %s cannot be listed (%v); nothing was imported", ResearchFolder, err)
|
||
}
|
||
var cands []*importCandidate
|
||
for _, d := range docs {
|
||
if adrFile.MatchString(d) {
|
||
cands = append(cands, &importCandidate{path: DecisionsFolder + "/" + d})
|
||
}
|
||
}
|
||
for _, f := range efforts {
|
||
if regexp.MustCompile(`^\d{3}-`).MatchString(f) {
|
||
cands = append(cands, &importCandidate{path: ResearchFolder + "/" + f})
|
||
}
|
||
}
|
||
if len(cands) == 0 {
|
||
// The records left hq (hq issue 426): what it held is here, and an import from it would say "nothing
|
||
// created" as a success (hq issue 488).
|
||
return nil, fmt.Errorf("hq holds no decision record or research effort at commit %s: its %s and %s hold none. "+
|
||
"The records live in the knowledge base now, and nothing was imported", short(commit), DecisionsFolder, ResearchFolder)
|
||
}
|
||
// Read and parse, a few at a time, every document at the one commit.
|
||
var moved error
|
||
var movedMu sync.Mutex
|
||
read := func(path string) (string, error) {
|
||
text, at, err := records.Read(path)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if at != commit {
|
||
movedMu.Lock()
|
||
moved = fmt.Errorf("hq moved while it was read: it stood at %s and %s was read at %s. Run the import again", short(commit), path, short(at))
|
||
movedMu.Unlock()
|
||
}
|
||
return text, nil
|
||
}
|
||
var wg sync.WaitGroup
|
||
sem := make(chan struct{}, importReaders)
|
||
for _, c := range cands {
|
||
wg.Add(1)
|
||
sem <- struct{}{}
|
||
go func(c *importCandidate) {
|
||
defer wg.Done()
|
||
defer func() { <-sem }()
|
||
if strings.HasPrefix(c.path, DecisionsFolder) {
|
||
text, err := read(c.path)
|
||
if err != nil {
|
||
c.err = fmt.Errorf("could not be read: %v", err)
|
||
return
|
||
}
|
||
c.parsed, c.err = ParseDecision(c.path, text)
|
||
return
|
||
}
|
||
_, names, err := records.List(c.path)
|
||
if err != nil {
|
||
c.err = fmt.Errorf("could not be listed: %v", err)
|
||
return
|
||
}
|
||
texts := map[string]string{}
|
||
for _, n := range names {
|
||
text, err := read(c.path + "/" + n)
|
||
if err != nil {
|
||
c.err = fmt.Errorf("%s could not be read: %v", n, err)
|
||
return
|
||
}
|
||
texts[n] = text
|
||
}
|
||
c.parsed, c.err = ParseResearch(docName(c.path), texts)
|
||
}(c)
|
||
}
|
||
wg.Wait()
|
||
if moved != nil {
|
||
return nil, fmt.Errorf("%v; nothing was imported", moved)
|
||
}
|
||
|
||
out := &Imported{DryRun: a.DryRun, Commit: commit, Rows: []ImportRow{}}
|
||
byKey := map[Key][]*importCandidate{}
|
||
for _, c := range cands {
|
||
if c.parsed != nil {
|
||
byKey[c.parsed.Record.Key()] = append(byKey[c.parsed.Record.Key()], c)
|
||
}
|
||
}
|
||
for key, cs := range byKey {
|
||
if len(cs) > 1 {
|
||
paths := make([]string, len(cs))
|
||
for i, c := range cs {
|
||
paths[i] = c.path
|
||
}
|
||
for _, c := range cs {
|
||
c.err = fmt.Errorf("%s is the number of %s: a number names one record", key, strings.Join(paths, " and "))
|
||
}
|
||
}
|
||
}
|
||
sort.Slice(cands, func(i, j int) bool { return cands[i].path < cands[j].path })
|
||
|
||
ctx := context.Background()
|
||
k.mu.Lock()
|
||
defer k.mu.Unlock()
|
||
known := map[Key]bool{}
|
||
for key := range byKey {
|
||
known[key] = true
|
||
}
|
||
all, err := k.Store.All(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("the knowledge base cannot be read (%v); nothing was imported", err)
|
||
}
|
||
held := map[Key]*Entry{}
|
||
for _, e := range all {
|
||
held[e.Current.Key()] = e
|
||
known[e.Current.Key()] = true
|
||
}
|
||
at := k.now()
|
||
var rows []ImportRow
|
||
for _, c := range cands {
|
||
row := ImportRow{From: c.path}
|
||
if c.err != nil {
|
||
row.Action, row.Why = ImportRefused, c.err.Error()
|
||
row.Record = c.path
|
||
out.Refused++
|
||
rows = append(rows, row)
|
||
continue
|
||
}
|
||
r := c.parsed.Record
|
||
row.Record, row.Status = r.Key().String(), r.Status
|
||
out.Read++
|
||
out.Odd = append(out.Odd, c.parsed.Odd...)
|
||
for i, l := range r.Links {
|
||
if to, ok := l.To.Record(); ok && !known[to] {
|
||
r.Links[i].Unresolved = true
|
||
if held[r.Key()] == nil {
|
||
out.Unresolved++
|
||
out.Odd = append(out.Odd, fmt.Sprintf("%s %s %s, which hq does not hold at this commit", r.Key(), l.Kind, to))
|
||
}
|
||
}
|
||
if l.Kind == LinkChanges && l.To.Clause == 0 {
|
||
r.Links[i].Source = LinkPending
|
||
}
|
||
}
|
||
r.Source = "hq@" + commit + ":" + c.path
|
||
if e := held[r.Key()]; e != nil {
|
||
if !a.Refresh {
|
||
row.Action = ImportHeld
|
||
out.Held++
|
||
continue
|
||
}
|
||
k.refresh(ctx, a, e, r, c.path, commit, caller, &row, out)
|
||
if row.Action != ImportHeld {
|
||
rows = append(rows, row)
|
||
}
|
||
continue
|
||
}
|
||
out.Clauses += len(r.Clauses)
|
||
if r.Drafting != nil {
|
||
out.ToDraft++
|
||
}
|
||
row.Action = ImportWould
|
||
if !a.DryRun {
|
||
err := k.Store.Create(ctx, Version{By: caller, At: at, Why: "imported from hq at " + short(commit) + ": " + c.path, Record: r})
|
||
switch {
|
||
case errors.Is(err, ErrExists):
|
||
row.Action = ImportHeld
|
||
out.Held++
|
||
continue
|
||
case err != nil:
|
||
row.Action, row.Why = ImportRefused, err.Error()
|
||
out.Refused++
|
||
rows = append(rows, row)
|
||
continue
|
||
}
|
||
row.Action = ImportCreate
|
||
k.emit(EventImported, r, caller, "imported from hq at "+short(commit), 1)
|
||
}
|
||
out.Created++
|
||
rows = append(rows, row)
|
||
}
|
||
if len(rows) > MostImportRows {
|
||
out.Cut = fmt.Sprintf("%d more row(s) are not shown; the counts are whole", len(rows)-MostImportRows)
|
||
rows = rows[:MostImportRows]
|
||
}
|
||
out.Rows = append(out.Rows, rows...)
|
||
if len(out.Odd) > 40 {
|
||
out.Odd = append(out.Odd[:40], fmt.Sprintf("and %d more", len(out.Odd)-40))
|
||
}
|
||
verb, again := "created", "refreshed"
|
||
if a.DryRun {
|
||
verb, again = "would create", "would refresh"
|
||
}
|
||
out.Said = fmt.Sprintf("Read %d records of hq at commit %s: %s %d, already held %d, refused %d; %d clauses written as such, "+
|
||
"%d decisions written as prose wait for their clauses to be drafted and confirmed.",
|
||
out.Read, short(commit), verb, out.Created, out.Held+out.Refreshed+out.Unchanged+out.ChangedSince, out.Refused, out.Clauses, out.ToDraft)
|
||
if a.Refresh {
|
||
out.Said += fmt.Sprintf(" Of those held: %s %d that only the import ever wrote, %d read again and the same, %d changed here "+
|
||
"since their import and left as they are.", again, out.Refreshed, out.Unchanged, out.ChangedSince)
|
||
}
|
||
switch {
|
||
case a.DryRun:
|
||
out.Said += " Nothing was written: a dry run."
|
||
case out.Created == 0 && out.Refreshed == 0 && out.Refused == 0:
|
||
out.Said += " Nothing changed: every record is already held as hq says it."
|
||
case out.Created == 0 && out.Refreshed == 0:
|
||
out.Said += " Nothing changed: every record not refused is already held."
|
||
}
|
||
if !a.DryRun && out.Created+out.Refreshed > 0 {
|
||
out.Findings = k.judgeSet(ctx)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// draftedHere is the why of a version the knowledge base wrote itself while drafting and confirming clauses, or
|
||
// judging which clause a change of hq's names (draft.go): the import's side of a record, not anyone's change.
|
||
var draftedHere = regexp.MustCompile(`^(?:\d+ clauses drafted by |\d+ of \d+ drafted clauses confirmed by |drafting failed: |` +
|
||
`confirming failed: |no Decision section|which clause of .+ it changes drafted: |that it changes .+ (?:was not confirmed: |confirmed by )|` +
|
||
`§\d+ changed by .+ \(as hq said, its clause judged and confirmed\)$)`)
|
||
|
||
// importedOnly says whether every version of a record was written by the import, a refresh of it, or the
|
||
// knowledge base's own drafting and confirming of its clauses: nobody has changed it since it was read from hq. A
|
||
// return, a correction, an acceptance over it or an insight is someone's change.
|
||
func importedOnly(vs []Version) bool {
|
||
if len(vs) == 0 {
|
||
return false
|
||
}
|
||
for _, v := range vs {
|
||
if v.Insight || v.Record == nil || !strings.HasPrefix(v.Record.Source, "hq@") {
|
||
return false
|
||
}
|
||
imported := strings.HasPrefix(v.Why, "imported from hq at ") || strings.HasPrefix(v.Why, "refreshed from hq at ")
|
||
drafted := draftedHere.MatchString(v.Why) && (v.By == "the knowledge base" || strings.HasPrefix(v.By, "the agent on "))
|
||
if !imported && !drafted {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// carry keeps in a record read again (now) what the knowledge base made of the one held: its clauses with their
|
||
// statuses when hq writes the same ones; its drafted or confirmed clauses and their drafting while the text they
|
||
// were drafted from is what hq says (else they are drafted again); and the clause a change of hq's names, once
|
||
// judged.
|
||
func carry(held, now *Record) {
|
||
switch {
|
||
case len(now.Clauses) > 0:
|
||
if len(held.Clauses) == len(now.Clauses) {
|
||
same := true
|
||
for i := range now.Clauses {
|
||
same = same && held.Clauses[i].N == now.Clauses[i].N && held.Clauses[i].Text == now.Clauses[i].Text
|
||
}
|
||
if same {
|
||
now.Clauses = held.Clauses
|
||
}
|
||
}
|
||
case now.Drafting != nil && held.Drafting != nil && (len(held.Clauses) > 0 || held.Drafting.State != DraftWaiting):
|
||
a, _ := draftSource(held)
|
||
b, _ := draftSource(now)
|
||
if a == b {
|
||
now.Clauses, now.Drafting = held.Clauses, held.Drafting
|
||
}
|
||
}
|
||
judged := map[Ref][]Link{}
|
||
for _, l := range held.Links {
|
||
if l.Kind == LinkChanges && l.Source != LinkPending {
|
||
if to, ok := l.To.Record(); ok {
|
||
judged[Ref{Kind: to.Kind, Number: to.Number}] = append(judged[Ref{Kind: to.Kind, Number: to.Number}], l)
|
||
}
|
||
}
|
||
}
|
||
var links []Link
|
||
for _, l := range now.Links {
|
||
target := Ref{Kind: l.To.Kind, Number: l.To.Number}
|
||
if l.Kind == LinkChanges && l.Source == LinkPending && len(judged[target]) > 0 {
|
||
for _, j := range judged[target] {
|
||
j.Unresolved = l.Unresolved
|
||
links = append(links, j)
|
||
}
|
||
delete(judged, target)
|
||
continue
|
||
}
|
||
links = append(links, l)
|
||
}
|
||
now.Links = links
|
||
}
|
||
|
||
// refresh reads a held record again (r, as hq has it at the commit) and, when only the import ever wrote it and
|
||
// what it holds differs, writes r as its new version naming what changed and why. A record changed here since
|
||
// its import is never touched; a second refresh finds the same and changes nothing.
|
||
func (k *KB) refresh(ctx context.Context, a ImportAsk, e *Entry, r *Record, path, commit, caller string, row *ImportRow, out *Imported) {
|
||
vs, err := k.Store.Versions(ctx, r.Key())
|
||
if err != nil {
|
||
row.Action, row.Why = ImportRefused, "its versions cannot be read: "+err.Error()
|
||
out.Refused++
|
||
return
|
||
}
|
||
if e.Pending != nil || !importedOnly(vs) {
|
||
last := vs[len(vs)-1]
|
||
row.Action = ImportChangedSince
|
||
row.Why = fmt.Sprintf("changed here since its import (version %d by %s: %s); a refresh never touches it", last.N, last.By, last.Why)
|
||
if e.Pending != nil {
|
||
row.Why = "a version waits for its review over it; a refresh never touches it"
|
||
}
|
||
out.ChangedSince++
|
||
return
|
||
}
|
||
carry(e.Current, r)
|
||
diffs := differences(e.Current, r)
|
||
if len(diffs) == 0 {
|
||
row.Action = ImportHeld
|
||
out.Unchanged++
|
||
return
|
||
}
|
||
row.Why = strings.Join(diffs, "; ")
|
||
row.Action = ImportWouldRefresh
|
||
out.Refreshed++
|
||
if a.DryRun {
|
||
return
|
||
}
|
||
why := fmt.Sprintf("refreshed from hq at %s: %s; the import before hq issue 487 had lost or misread %s", short(commit), path, row.Why)
|
||
if len(why) > 1000 {
|
||
why = why[:997] + "..."
|
||
}
|
||
if err := k.Store.Save(ctx, Version{By: caller, At: k.now(), Why: why, Record: r}, true, false, e.Revision); err != nil {
|
||
row.Action, row.Why = ImportRefused, "it could not be written: "+err.Error()
|
||
out.Refused++
|
||
out.Refreshed--
|
||
return
|
||
}
|
||
row.Action = ImportRefresh
|
||
k.emit(EventChanged, r, caller, why, e.Versions+1)
|
||
}
|
||
|
||
// differences names what a record read again from hq holds that the one held does not, field by field; none when
|
||
// the two hold the same.
|
||
func differences(old, now *Record) []string {
|
||
var out []string
|
||
o, n := old.Clone(), now.Clone()
|
||
o.Source, n.Source = "", ""
|
||
if a, b := renderJSON(o), renderJSON(n); a == b {
|
||
return nil
|
||
}
|
||
field := func(name, a, b string) {
|
||
if a != b {
|
||
if a == "" {
|
||
a = "unset"
|
||
}
|
||
if b == "" {
|
||
b = "unset"
|
||
}
|
||
out = append(out, fmt.Sprintf("%s: %s → %s", name, a, b))
|
||
}
|
||
}
|
||
boolOf := func(p *bool) string {
|
||
if p == nil {
|
||
return ""
|
||
}
|
||
return fmt.Sprint(*p)
|
||
}
|
||
field("title", o.Title, n.Title)
|
||
field("status", o.Status, n.Status)
|
||
field("topic", o.Topic, n.Topic)
|
||
field("date", o.Date, n.Date)
|
||
field("deciders", strings.Join(o.Deciders, ", "), strings.Join(n.Deciders, ", "))
|
||
field("reconstructed", boolOf(o.Reconstructed), boolOf(n.Reconstructed))
|
||
field("amended", strings.Join(o.Amended, ", "), strings.Join(n.Amended, ", "))
|
||
field("touches", strings.Join(o.Touches, ", "), strings.Join(n.Touches, ", "))
|
||
var gone []string
|
||
for key := range o.Extra {
|
||
if _, ok := n.Extra[key]; !ok {
|
||
gone = append(gone, key)
|
||
}
|
||
}
|
||
sort.Strings(gone)
|
||
if len(gone) > 0 {
|
||
out = append(out, "held as a field of its own now: "+strings.Join(gone, ", "))
|
||
}
|
||
if renderJSON(o.Clauses) != renderJSON(n.Clauses) {
|
||
s := fmt.Sprintf("clauses: %d → %d", len(o.Clauses), len(n.Clauses))
|
||
if len(n.Clauses) == 0 && n.Drafting != nil {
|
||
s += ", to be drafted"
|
||
}
|
||
out = append(out, s)
|
||
}
|
||
if (o.Drafting == nil) != (n.Drafting == nil) || (n.Drafting != nil && o.Drafting != nil && o.Drafting.Reason != n.Drafting.Reason) {
|
||
if n.Drafting != nil {
|
||
s := "drafting queued"
|
||
if n.Drafting.Reason != "" {
|
||
s += " (" + n.Drafting.Reason + ")"
|
||
}
|
||
out = append(out, s)
|
||
} else {
|
||
out = append(out, "no drafting: its clauses are written")
|
||
}
|
||
}
|
||
ol, nl := map[string]bool{}, map[string]bool{}
|
||
for _, l := range o.Links {
|
||
ol[l.String()] = true
|
||
}
|
||
for _, l := range n.Links {
|
||
nl[l.String()] = true
|
||
}
|
||
var added, removed []string
|
||
for l := range nl {
|
||
if !ol[l] {
|
||
added = append(added, l)
|
||
}
|
||
}
|
||
for l := range ol {
|
||
if !nl[l] {
|
||
removed = append(removed, l)
|
||
}
|
||
}
|
||
sort.Strings(added)
|
||
sort.Strings(removed)
|
||
if len(added)+len(removed) > 0 {
|
||
s := "links:"
|
||
if len(added) > 0 {
|
||
s += " + " + strings.Join(added, ", + ")
|
||
}
|
||
if len(removed) > 0 {
|
||
s += " − " + strings.Join(removed, ", − ")
|
||
}
|
||
out = append(out, s)
|
||
}
|
||
if renderJSON(o.Sections) != renderJSON(n.Sections) {
|
||
out = append(out, "its sections")
|
||
}
|
||
if len(out) == 0 {
|
||
out = append(out, "what it holds otherwise (as hq writes it)")
|
||
}
|
||
return out
|
||
}
|
||
|
||
func renderJSON(v any) string {
|
||
raw, _ := json.Marshal(v)
|
||
return string(raw)
|
||
}
|