diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/form.go b/modules/mesh-knowledge/cmd/mesh-knowledge/form.go new file mode 100644 index 000000000..cd8be6f8a --- /dev/null +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/form.go @@ -0,0 +1,262 @@ +package main + +// A form-only version (novox/hq to-be 55 §2.2, ADR 0592 §31): the records imported with file-named sections get plain titles +// in one version per record, written after the move by the module's tool alone. It changes only the sections' names and the +// targets of links to files, each becoming a reference of the one form. It is marked `form`, lists the names it changed, +// and is accepted without a judgement because a check proves only form changed: each section's text with every link replaced +// by its label is the same before and after, and each new link target names the record the old one named. `get` takes the +// old section names as aliases, recorded in the version. A part's review carries over to the part renamed. + +import ( + "context" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + "unicode" +) + +// FormTool is the module's tool that lists, dry-runs and runs the form-only versions. +const FormTool = "knowledge_form_only" + +var ( + fileLink = regexp.MustCompile(`\[([^\]]*)\]\(([^)\s]+)\)`) + issueTarget = regexp.MustCompile(`04-ISSUES/(\d{3,5})-`) +) + +// refTarget is the target a reference is written as in a link: the scheme ref: and the reference of the one form. +func refTarget(ref string) string { return "ref:" + url.PathEscape(ref) } + +// targetKey is the record a link's target names, in the old form (a file) or the new (a ref:). +func targetRef(target string) (string, bool) { + if rest, ok := strings.CutPrefix(target, "ref:"); ok { + s, err := url.PathUnescape(rest) + return s, err == nil + } + switch { + case researchFolder.MatchString(target): + return fmt.Sprintf("research %03d", atoi(researchFolder.FindStringSubmatch(target)[1])), true + case issueTarget.MatchString(target): + return fmt.Sprintf("issue %d", atoi(issueTarget.FindStringSubmatch(target)[1])), true + case designPath.MatchString(target): + return "design " + designPath.FindString(target), true + case adrFile.MatchString(target): + return fmt.Sprintf("ADR %04d", atoi(adrFile.FindStringSubmatch(target)[1])), true + } + return "", false +} + +// titleOf is a plain title for a file-named section: "00-overview.md" is "Overview", "01-open-questions.md" "Open questions". +func titleOf(name string) string { + t := name + if i := strings.LastIndexByte(t, '/'); i >= 0 { + t = t[i+1:] + } + t = fileExtension.ReplaceAllString(t, "") + t = numberPrefix.ReplaceAllString(t, "") + t = strings.NewReplacer("-", " ", "_", " ").Replace(strings.TrimSpace(t)) + if t == "" { + return name + } + r := []rune(t) + r[0] = unicode.ToUpper(r[0]) + return string(r) +} + +// FormChange is what a form-only version changes. +type FormChange struct { + Renamed map[string]string `json:"renamed,omitempty"` + Relinked int `json:"relinked"` +} + +// formOf is the record with plain titles and links of the one form, and what it changed; none when it is already so. +func formOf(r *Record) (*Record, *FormChange) { + n := r.Clone() + ch := &FormChange{Renamed: map[string]string{}} + taken := map[string]bool{} + for _, s := range n.Sections { + if len(sectionNameProblems(s.Name)) == 0 { + taken[s.Name] = true + } + } + for i, s := range n.Sections { + if len(sectionNameProblems(s.Name)) > 0 { + t := titleOf(s.Name) + for base, k := t, 2; taken[t]; k++ { + t = fmt.Sprintf("%s %d", base, k) + } + taken[t] = true + ch.Renamed[s.Name] = t + n.Sections[i].Name = t + } + n.Sections[i].Text = fileLink.ReplaceAllStringFunc(n.Sections[i].Text, func(link string) string { + m := fileLink.FindStringSubmatch(link) + if strings.HasPrefix(m[2], "ref:") || !linkToFile(m[2]) { + return link + } + ref, ok := targetRef(m[2]) + if !ok { + return link + } + ch.Relinked++ + return "[" + m[1] + "](" + refTarget(ref) + ")" + }) + } + if len(ch.Renamed) == 0 && ch.Relinked == 0 { + return r, nil + } + if len(ch.Renamed) > 0 { + if n.Aliases == nil { + n.Aliases = map[string]string{} + } + for old, t := range ch.Renamed { + n.Aliases[old] = t + } + } + return n, ch +} + +// checkForm proves that only form changed between two versions: the fields and clauses and links are the same, each +// section's text with every link replaced by its label is the same, and each link's new target names what the old one did. +func checkForm(old, nw *Record, ch *FormChange) error { + if len(old.Sections) != len(nw.Sections) { + return fmt.Errorf("the number of sections changed") + } + strip := func(r *Record) string { + c := r.Clone() + c.Sections, c.Aliases, c.Parts, c.FormOnly = nil, nil, nil, false + return Render(c) + } + if strip(old) != strip(nw) { + return fmt.Errorf("a field, a clause or a link changed") + } + for i := range old.Sections { + label := func(s string) string { return fileLink.ReplaceAllString(s, "$1") } + if label(old.Sections[i].Text) != label(nw.Sections[i].Text) { + return fmt.Errorf("the text of %q changed beyond its links", old.Sections[i].Name) + } + o, n := fileLink.FindAllStringSubmatch(old.Sections[i].Text, -1), fileLink.FindAllStringSubmatch(nw.Sections[i].Text, -1) + for j := range o { + ro, ok1 := targetRef(o[j][2]) + rn, ok2 := targetRef(n[j][2]) + if ok1 && o[j][2] != n[j][2] && (!ok2 || ro != rn) { + return fmt.Errorf("the link %q of %q names %s, and its new target %s", o[j][1], old.Sections[i].Name, ro, rn) + } + } + } + if len(ch.Renamed) == 0 && ch.Relinked == 0 { + return fmt.Errorf("nothing changed") + } + return nil +} + +// FormedRecord is one record a form-only version was written (or would be) for. +type FormedRecord struct { + Record string `json:"record"` + Renamed map[string]string `json:"renamed,omitempty"` + Relinked int `json:"relinked,omitempty"` + Skipped string `json:"skipped,omitempty"` +} + +// FormOnly lists (dry), or writes (commit), the form-only version of one record or of every record that needs one. A +// version is written only when the check proves form alone changed; a record with a version waiting for its review is +// skipped, saying so. It answers what it did, record by record. +func (k *KB) FormOnly(caller string, only *Key, commit bool) ([]FormedRecord, error) { + if _, err := k.ledger(); err != nil { + return nil, err + } + if commit && !proven(caller) { + return nil, fmt.Errorf("a form-only version needs a caller the runtime proved; this call is by %s", caller) + } + k.mu.Lock() + defer k.mu.Unlock() + ctx := k.ctxFor(caller) + all, err := k.Store.All(ctx) + if err != nil { + return nil, err + } + var out []FormedRecord + for _, e := range all { + key := e.Current.Key() + if only != nil && *only != key { + continue + } + next, ch := formOf(e.Current) + if ch == nil { + continue + } + fr := FormedRecord{Record: key.String(), Renamed: ch.Renamed, Relinked: ch.Relinked} + switch { + case e.Pending != nil: + fr.Skipped = "a version waits for its review: it takes the form-only version once that is decided" + case e.Current.Status == StatusReserved || e.Current.Status == StatusWithdrawn: + fr.Skipped = "it is " + e.Current.Status + default: + if err := checkForm(e.Current, next, ch); err != nil { + fr.Skipped = "the check refused it: " + err.Error() + } else if commit { + k.carryParts(next, ch) + next.FormOnly = true + names := make([]string, 0, len(ch.Renamed)) + for old, t := range ch.Renamed { + names = append(names, old+" → "+t) + } + sort.Strings(names) + why := fmt.Sprintf("form only: %d section name(s), %d link(s) written in the one form", len(ch.Renamed), ch.Relinked) + if len(names) > 0 { + why += " (" + strings.Join(names, "; ") + ")" + } + if err := k.Store.Save(ctx, Version{By: caller, At: k.now(), Why: why, Record: next}, true, false, e.Revision); err != nil { + fr.Skipped = "it could not be written: " + err.Error() + } else { + next.FormOnly = false + k.emit(EventFormed, next, caller, why, e.Versions+1) + } + } + } + out = append(out, fr) + } + return out, nil +} + +// carryParts moves a record's part verdicts to the parts renamed, with the digest of the new part, so the review carries +// over to a version whose meaning is unchanged. +func (k *KB) carryParts(r *Record, ch *FormChange) { + if len(r.Parts) == 0 { + return + } + linked := k.linkedOf(context.Background(), r) + fresh := map[string]string{} + for _, p := range partsOf(r, linked) { + fresh[p.name] = p.digest + } + var out []PartReview + for _, p := range r.Parts { + if t, ok := ch.Renamed[p.Part]; ok { + p.Part = t + } + if d, ok := fresh[p.Part]; ok { + p.Digest = d + out = append(out, p) + } + } + r.Parts = out +} + +// linkedOf are the records r links, as the review reads them. +func (k *KB) linkedOf(ctx context.Context, r *Record) []*Record { + var linked []*Record + seen := map[Key]bool{r.Key(): true} + for _, l := range r.Links { + to, ok := l.To.Record() + if !ok || seen[to] { + continue + } + seen[to] = true + if t, err := k.Store.Get(ctx, to); err == nil && t != nil { + linked = append(linked, t.Current) + } + } + return linked +} diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/form_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/form_test.go new file mode 100644 index 000000000..b92ac2d5e --- /dev/null +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/form_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" +) + +// A form-only version (to-be 55 §2.2, AC55): file-named sections get plain titles and links to files become references of +// the one form, in one version a check proves changes only form; the old names stay as aliases; a part's review carries over; +// nothing is written without commit, and a record waiting for its review is skipped. +func TestAFormOnlyVersionChangesTheNamesAndTheLinksAndNothingElse(t *testing.T) { + w, _ := ledgerWorld(t) + r, _ := w.k.ReserveNumber("author@g14", KindResearch, "an effort") + old := effortAsk(r.Number, StatusActive, + Section{Name: "00-overview.md", Text: "What it is. See [the window](0276-merges-are-assembled-in-a-rolling-window-and-each-batch-is-delivered-by-one-walk-with-one-commit-per-repository.md) and [the plan](https://example.org/plan.md)."}, + Section{Name: "01-open-questions.md", Text: "Is it [ready](../../01-RESEARCH/045-a-knowledge-base-the-mesh-writes/01-findings.md)?"}) + w.accepted(t, "author@g14", old) + key := Key{KindResearch, r.Number} + before := mustGet(t, w.k.Store, key) + w.k.Store.(*ledgerStore).mu.Lock() + versions := before.Versions + w.k.Store.(*ledgerStore).mu.Unlock() + + dry, err := w.k.FormOnly("lead@g14", &key, false) + if err != nil || len(dry) != 1 || dry[0].Renamed["00-overview.md"] != "Overview" || dry[0].Renamed["01-open-questions.md"] != "Open questions" || dry[0].Relinked != 2 { + t.Fatalf("%v %+v", err, dry) + } + if e := mustGet(t, w.k.Store, key); e.Versions != versions { + t.Fatal("a dry run wrote a version") + } + if _, err := w.k.FormOnly("", &key, true); err == nil { + t.Fatal("an unproven caller wrote a form-only version") + } + done, err := w.k.FormOnly("lead@g14", &key, true) + if err != nil || len(done) != 1 || done[0].Skipped != "" { + t.Fatalf("%v %+v", err, done) + } + e := mustGet(t, w.k.Store, key) + if e.Versions != versions+1 || e.Current.Status != StatusActive || e.Pending != nil || e.Current.Judging != nil { + t.Fatalf("%d versions, %s", e.Versions, e.Current.Status) + } + got := e.Current + if got.Sections[0].Name != "Overview" || got.Sections[1].Name != "Open questions" || + !strings.Contains(got.Sections[0].Text, "[the window](ref:ADR%200276)") || !strings.Contains(got.Sections[0].Text, "(https://example.org/plan.md)") || + !strings.Contains(got.Sections[1].Text, "[ready](ref:research%20045)") { + t.Fatalf("%+v", got.Sections) + } + // get takes the old name as an alias, and the review of the parts carries over to the renamed parts. + if got.Section("00-overview.md") != got.Sections[0].Text || got.Aliases["01-open-questions.md"] != "Open questions" { + t.Fatalf("aliases %v", got.Aliases) + } + if len(got.Parts) != 3 || got.Parts[1].Part != "Overview" { + t.Fatalf("parts %+v", got.Parts) + } + for _, p := range partsOf(got, w.k.linkedOf(t.Context(), got)) { + if keptPart(got, p) == nil { + t.Errorf("the part %q is judged again after a change of form", p.name) + } + } + h, _ := w.k.History(key, 0, 0, 0) + if last := h.Versions[len(h.Versions)-1]; !strings.HasPrefix(last.Why, "form only:") { + t.Fatalf("%+v", last) + } + if len(w.said.of(EventFormed)) != 1 { + t.Fatalf("events %v", w.said.events) + } + // Done: another run has nothing left to do; and the check refuses a change that is more than form. + if again, _ := w.k.FormOnly("lead@g14", &key, true); len(again) != 0 { + t.Fatalf("%+v", again) + } + more := e.Current.Clone() + more.Sections[0].Text += " And more." + if err := checkForm(e.Current, more, &FormChange{Relinked: 1}); err == nil { + t.Fatal("a change of text passed as form") + } + moreLinks := old + _ = moreLinks + wrong, ch := formOf(&Record{Kind: KindResearch, Number: 1, Sections: []Section{{Name: "a.md", Text: "[x](0276-a.md)"}}}) + wrong.Sections[0].Text = "[x](ref:ADR%200131)" + if err := checkForm(&Record{Kind: KindResearch, Number: 1, Sections: []Section{{Name: "a.md", Text: "[x](0276-a.md)"}}}, wrong, ch); err == nil { + t.Fatal("a link to another record passed as form") + } +} diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/record.go b/modules/mesh-knowledge/cmd/mesh-knowledge/record.go index 5c217c1ad..0aab1615c 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/record.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/record.go @@ -329,7 +329,11 @@ type Record struct { Reviews []Review `json:"reviews,omitempty"` // Parts are the latest verdict of each part of the version (the head, each section), kept for a new version whose // part is unchanged (review_parts.go). - Parts []PartReview `json:"parts,omitempty"` + Parts []PartReview `json:"parts,omitempty"` + // Aliases are the file names the sections had before their form-only version, by old name; get takes them for the title. + Aliases map[string]string `json:"aliases,omitempty"` + // FormOnly marks the version that changed only the form (to-be 55 §2.2). + FormOnly bool `json:"form,omitempty"` Ask *OperatorAsk `json:"ask,omitempty"` Drafting *Drafting `json:"drafting,omitempty"` // Judging is where the version proposed stands in the knowledge base's own review: due, or failed and why. @@ -440,6 +444,9 @@ func (r *Record) Section(name string) string { return s.Text } } + if t, ok := r.Aliases[name]; ok { + return r.Section(t) + } return "" } @@ -465,7 +472,7 @@ func (r *Record) Clone() *Record { // it. func (r *Record) Digest() string { c := r.Clone() - c.Reviews, c.Ask, c.Drafting, c.Judging, c.Parts = nil, nil, nil, nil, nil + c.Reviews, c.Ask, c.Drafting, c.Judging, c.Parts, c.FormOnly = nil, nil, nil, nil, nil, false raw, _ := json.Marshal(c) sum := sha256.Sum256(raw) return "sha256:" + hex.EncodeToString(sum[:]) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/sectionform.go b/modules/mesh-knowledge/cmd/mesh-knowledge/sectionform.go index e065927d7..d0fa7652e 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/sectionform.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/sectionform.go @@ -29,7 +29,7 @@ var ( // linkToFile says whether a link's target is a relative path or a file, not an address or an anchor. func linkToFile(target string) bool { switch { - case strings.HasPrefix(target, "#"), strings.HasPrefix(target, "mailto:"): + case strings.HasPrefix(target, "#"), strings.HasPrefix(target, "mailto:"), strings.HasPrefix(target, "ref:"): return false case regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*://`).MatchString(target): return false diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/tools.go b/modules/mesh-knowledge/cmd/mesh-knowledge/tools.go index 6a9ba1a13..5ec1ec221 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/tools.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/tools.go @@ -33,7 +33,7 @@ const ( // ModuleTools are the module's own tools beside the health tool, in the order the manifest lists them. var ModuleTools = []string{HealthTool, HighestTool, MarksTool, HeadTool, OperationsTool, SectionTool, DraftTool, PartsTool, - AskedTool, FillTool, IndexTool, AgainTool, IntegrityTool, MoveTool, MoveCheckTool} + AskedTool, FillTool, IndexTool, AgainTool, IntegrityTool, FormTool, MoveTool, MoveCheckTool} var errNotOnLedger = errors.New("the records are still in the database: this tool reads the ledger, which serves them after the move") @@ -321,6 +321,35 @@ func moduleTools(k *KB, caller func(map[string]any) string) []stdio.Tool { } return map[string]any{"checked": len(keys), "damaged": bad}, nil }}, + {Name: FormTool, + Description: "List, or with commit write, the form-only versions: a record imported with file-named sections gets plain titles, and its links to files " + + "become references of the one form, in one version that changes nothing else. A check proves only form changed (each section's text with its links " + + "replaced by their labels is the same, each link names the same record); the old section names stay as aliases. Without commit nothing is written. " + + "Takes one record or every record that needs one. Refused while the records are in the database.", + Input: schema(nil, map[string]any{"kind": kindArg(), "number": integer("one record; every record that needs a version when absent"), "commit": flag("write the versions")}), + Run: func(m map[string]any) (any, error) { + a, err := readArgs(FormTool, m, "kind", "number", "commit") + if err != nil { + return nil, err + } + var only *Key + if _, has := m["number"]; has { + key, err := a.key() + if err != nil { + return nil, err + } + only = &key + } + commit, err := a.flag("commit") + if err != nil { + return nil, err + } + done, err := k.FormOnly(caller(m), only, commit) + if err != nil { + return nil, err + } + return map[string]any{"records": done, "written": commit}, nil + }}, {Name: MoveTool, Description: "Move the records from the database to the ledger: with dry-run, say what a copy would write and write nothing; with commit, refuse writes for " + "its length, copy every version of every record, check the copy record by record, and set the records to be served from the ledger only if the check is " + diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/tools_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/tools_test.go index b8c9dbf05..f45661786 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/tools_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/tools_test.go @@ -130,7 +130,7 @@ func TestEveryStateTheModuleKeepsHasAToolThatReadsIt(t *testing.T) { if _, err := callTool(t, tools, MoveCheckTool, map[string]any{}); err == nil { t.Fatal("a move check without a database") } - if len(ModuleTools) != 15 { + if len(ModuleTools) != 16 { t.Fatalf("%v", ModuleTools) } } diff --git a/modules/mesh-knowledge/module.json b/modules/mesh-knowledge/module.json index 436559a3f..650b57383 100644 --- a/modules/mesh-knowledge/module.json +++ b/modules/mesh-knowledge/module.json @@ -162,6 +162,7 @@ "knowledge_rebuild_index", "knowledge_review_again", "knowledge_integrity", + "knowledge_form_only", "knowledge_move", "knowledge_move_check" ],