records: port to Go, and keep the checkout in a directory it owns
Every sync had failed since a container that ran as root left the checkout root's: git refused it as dubious ownership, and records answered from a stale copy. The Go bundle clones into repository/ under its directory, clears the old layout where it can and names what it cannot. Drops the container- runtime capability the move into the runtime left behind. hq issue 251.
This commit is contained in:
jochen
@@ -39,3 +39,12 @@ says so in its log.
|
||||
Search the mesh, through the console, for a phrase that appears only in one design document here, and
|
||||
get it back. `records_search {"query": "…"}` is that search; its test does the same against a
|
||||
repository it makes.
|
||||
|
||||
## Where the checkout lives
|
||||
|
||||
A Go bundle the node's runtime launches as the operator account (`cmd/records`). It clones into
|
||||
`repository/` inside the directory the mesh gives it — a directory it makes, and so owns. An earlier
|
||||
layout cloned into the given directory itself, from a container running as root, and left files the
|
||||
operator account cannot change; git then refused every sync as "dubious ownership" (hq issue 251). What
|
||||
that layout left is removed where it is the module's, and named in `records_status` (`leftBehind`, with
|
||||
the one command that deletes it) where it is not.
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
// records: the mesh's record — decisions, designs and issues — read where it is written (novox/hq ADR 0025,
|
||||
// ADR 0153). A Go bundle the node's runtime launches; it keeps a checkout of one repository current on every
|
||||
// merge the forge announces and on a timer, and answers five questions about it. stdout is the MCP channel;
|
||||
// what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func readJSON(path string, into any) error {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, into)
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string { s, _ := a[k].(string); return s }
|
||||
|
||||
func tools(r *Records) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "records_search",
|
||||
Description: "Where a phrase appears in the decisions, designs and issues, as written: document, line, nearest heading. " +
|
||||
"Search the literal words of a symptom or a term before forming a hypothesis; the answer names the commit it was read at.",
|
||||
Input: map[string]any{
|
||||
"query": str("the phrase, matched case-insensitively as written"),
|
||||
"limit": map[string]any{"type": "number", "description": "at most this many places (default 50)"},
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
limit := 0
|
||||
if v, ok := a["limit"].(float64); ok {
|
||||
limit = int(v)
|
||||
}
|
||||
return r.Search(strArg(a, "query"), limit)
|
||||
}},
|
||||
{Name: "records_read",
|
||||
Description: "One document, whole, by its path in the repository — a decision record, a design document, an issue report.",
|
||||
Input: map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md")},
|
||||
Run: func(a map[string]any) (any, error) { return r.Read(strArg(a, "path")) }},
|
||||
{Name: "records_list",
|
||||
Description: "What a folder of the repository holds: its sub-folders and its documents. The root when no folder is named.",
|
||||
Input: map[string]any{"folder": str("a folder inside the repository (optional)")},
|
||||
Run: func(a map[string]any) (any, error) { return r.List(strArg(a, "folder")) }},
|
||||
{Name: "records_status",
|
||||
Description: "Where the checkout stands: the repository, the forge it is read from, the commit and its date, when it was last brought up to date.",
|
||||
Run: func(map[string]any) (any, error) { return r.Standing(), nil }},
|
||||
{Name: "records_sync",
|
||||
Description: "Bring the checkout up to date now, and say where it stands.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
r.Sync()
|
||||
return r.Standing(), nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// keep keeps the checkout current: at start, on every merge the forge announces into this repository, and on
|
||||
// an unhurried timer for the merges it did not hear about — a restart during a merge, a repository the forge
|
||||
// does not emit for. The record changes when thinking changes, not by the minute.
|
||||
func keep(r *Records) {
|
||||
r.Sync()
|
||||
s := r.Standing()
|
||||
commit := s.Commit
|
||||
if len(commit) > 8 {
|
||||
commit = commit[:8]
|
||||
}
|
||||
line := fmt.Sprintf("[records] %s at %s, %d document(s)", s.Repository, commit, s.Documents)
|
||||
if s.LastError != "" {
|
||||
line += " — " + s.LastError
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, line)
|
||||
if s.LeftBehind != "" {
|
||||
fmt.Fprintln(os.Stderr, "[records] "+s.LeftBehind)
|
||||
}
|
||||
|
||||
go func() {
|
||||
for range time.Tick(10 * time.Minute) {
|
||||
r.Sync()
|
||||
}
|
||||
}()
|
||||
|
||||
// A merge on the forge into the repository this reads: pull now. The event names the repository by
|
||||
// owner and name (gitea's `pull.merged`); anything else is somebody else's merge.
|
||||
subscribe := func() error {
|
||||
return stdio.Subscribe("gitea.pull.merged", func(e stdio.Envelope) error {
|
||||
var body struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
}
|
||||
_ = json.Unmarshal(e.Body, &body)
|
||||
if body.Owner+"/"+body.Repo != r.Repository {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[records] %s merged; syncing\n", r.Repository)
|
||||
r.Sync()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := subscribe()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "[records] hearing merges not yet (%v); asking again in %s\n", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
r, err := FromEnv(os.Getenv)
|
||||
if err != nil {
|
||||
// Without a repository to read there is nothing to answer: no tools rather than five that fail.
|
||||
fmt.Fprintf(os.Stderr, "[records] no tools — %v\n", err)
|
||||
if err := stdio.Serve("", nil); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
go keep(r)
|
||||
if err := stdio.Serve("", tools(r)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
package main
|
||||
|
||||
// The record, read where it is written (novox/hq ADR 0025, ADR 0153).
|
||||
//
|
||||
// A repository of decisions, designs and issues — markdown, nothing else — cloned from the mesh's own forge
|
||||
// and kept current. **A checkout, not a copy**: the same bytes the repository holds, at a commit every
|
||||
// answer names, refreshed on every merge the forge announces and on a timer besides. Nothing is
|
||||
// transformed, indexed or summarised on the way, so there is nothing that can drift from the source except
|
||||
// by lagging behind it, and the lag is a number in every answer.
|
||||
//
|
||||
// **The clone is in a directory this module made** (novox/hq issue 251). The module is given a directory;
|
||||
// it clones into `repository/` inside it, a directory it creates and so owns. The TypeScript module cloned
|
||||
// into the given directory itself, and a sync that once ran as another account left every file there that
|
||||
// account's: git refused the checkout as "dubious ownership" from then on, and the operator account the
|
||||
// module runs as could change none of it. What the old layout left is removed where it is this module's,
|
||||
// and named in the standing where it is not.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MostHits is the most places a search answers; a phrase found more often is a phrase to narrow.
|
||||
const MostHits = 50
|
||||
|
||||
// MostBytes is how much of a document is answered; a longer one is answered in part, and says so.
|
||||
const MostBytes = 200_000
|
||||
|
||||
// Hit is one place a phrase was found.
|
||||
type Hit struct {
|
||||
// Path is the document, relative to the repository's root.
|
||||
Path string `json:"path"`
|
||||
// Line is the line it was found on, from 1.
|
||||
Line int `json:"line"`
|
||||
// Heading is the nearest heading above it, so a hit reads as where in the document it is.
|
||||
Heading string `json:"heading"`
|
||||
// Text is the line itself, trimmed.
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// Standing is where the checkout stands.
|
||||
type Standing struct {
|
||||
Repository string `json:"repository"`
|
||||
Origin string `json:"origin"`
|
||||
// Commit is the commit the checkout is at, or empty before the first clone.
|
||||
Commit string `json:"commit"`
|
||||
// Committed is when that commit was made, as the repository says.
|
||||
Committed string `json:"committed"`
|
||||
// Fetched is when this reader last brought the checkout up to date.
|
||||
Fetched string `json:"fetched"`
|
||||
// Documents is how many markdown documents the checkout holds.
|
||||
Documents int `json:"documents"`
|
||||
// LastError is why the last sync failed, if it did; the checkout stands where it was.
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
// LeftBehind names what an earlier layout left in the module's directory that this account cannot
|
||||
// remove: another account's files, harmless, and the operator's to delete.
|
||||
LeftBehind string `json:"leftBehind,omitempty"`
|
||||
}
|
||||
|
||||
// Records reads one repository's checkout.
|
||||
type Records struct {
|
||||
// Base is the directory the mesh gives the module; the checkout is Base/repository.
|
||||
Base string
|
||||
// Origin is the forge's address, `scheme://host:port`, from the git provision's binding.
|
||||
Origin string
|
||||
// Repository is the repository's path on it, `owner/name`, from this module's settings.
|
||||
Repository string
|
||||
// URL overrides the clone URL; a test points it at a local repository.
|
||||
URL string
|
||||
|
||||
mu sync.Mutex
|
||||
syncing chan struct{}
|
||||
fetched string
|
||||
lastError string
|
||||
leftBehind string
|
||||
}
|
||||
|
||||
// Dir is where the checkout is.
|
||||
func (r *Records) Dir() string { return filepath.Join(r.Base, "repository") }
|
||||
|
||||
// CloneURL is the forge and the repository. Public repositories only; a credential would be a secret this
|
||||
// module has not asked for.
|
||||
func (r *Records) CloneURL() string {
|
||||
if r.URL != "" {
|
||||
return r.URL
|
||||
}
|
||||
return strings.TrimRight(r.Origin, "/") + "/" + r.Repository + ".git"
|
||||
}
|
||||
|
||||
func git(args ...string) (string, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("git %s: %s", strings.Join(args, " "), strings.TrimSpace(string(out)))
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// Sync brings the checkout up to date, cloning it if it does not exist. One at a time: a call while one
|
||||
// runs waits for that one rather than racing it. It never fails: a failed sync is recorded in the standing
|
||||
// and the checkout stands where it was, which is still an answer.
|
||||
func (r *Records) Sync() {
|
||||
r.mu.Lock()
|
||||
if r.syncing != nil {
|
||||
wait := r.syncing
|
||||
r.mu.Unlock()
|
||||
<-wait
|
||||
return
|
||||
}
|
||||
done := make(chan struct{})
|
||||
r.syncing = done
|
||||
r.mu.Unlock()
|
||||
|
||||
err := r.doSync()
|
||||
|
||||
r.mu.Lock()
|
||||
if err != nil {
|
||||
r.lastError = err.Error()
|
||||
fmt.Fprintf(os.Stderr, "[records] could not sync %s: %v\n", r.CloneURL(), err)
|
||||
} else {
|
||||
r.lastError = ""
|
||||
r.fetched = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
r.syncing = nil
|
||||
r.mu.Unlock()
|
||||
close(done)
|
||||
}
|
||||
|
||||
func (r *Records) doSync() error {
|
||||
r.clearOldLayout()
|
||||
dir := r.Dir()
|
||||
if _, err := os.Stat(filepath.Join(dir, ".git")); err != nil {
|
||||
_ = os.RemoveAll(dir) // a clone that never finished
|
||||
if err := os.MkdirAll(r.Base, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := git("clone", "--quiet", "--depth", "50", r.CloneURL(), dir)
|
||||
return err
|
||||
}
|
||||
// The checkout is the mesh's, so a local change is nobody's: reset to what the forge has, rather than
|
||||
// merging into something a hand may have touched.
|
||||
if _, err := git("-C", dir, "fetch", "--quiet", "--depth", "50", "origin"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := git("-C", dir, "reset", "--quiet", "--hard", "origin/HEAD")
|
||||
return err
|
||||
}
|
||||
|
||||
// clearOldLayout removes what the TypeScript module's layout left in Base — a checkout in Base itself —
|
||||
// where it is this account's, and remembers what it is not.
|
||||
func (r *Records) clearOldLayout() {
|
||||
entries, err := os.ReadDir(r.Base)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
foreign := 0
|
||||
for _, e := range entries {
|
||||
if e.Name() == "repository" {
|
||||
continue
|
||||
}
|
||||
full := filepath.Join(r.Base, e.Name())
|
||||
if err := os.RemoveAll(full); err != nil {
|
||||
foreign++
|
||||
}
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.leftBehind = ""
|
||||
if foreign > 0 {
|
||||
owner := "another account"
|
||||
if info, err := os.Stat(filepath.Join(r.Base, ".git")); err == nil {
|
||||
if st, ok := info.Sys().(*syscall.Stat_t); ok && st.Uid == 0 {
|
||||
owner = "root"
|
||||
}
|
||||
}
|
||||
r.leftBehind = fmt.Sprintf("%d entr(ies) of an earlier checkout in %s belong to %s and cannot be removed by this module; "+
|
||||
"they are not read — delete them once: sudo find %s -mindepth 1 -maxdepth 1 ! -name repository -exec rm -rf {} +",
|
||||
foreign, r.Base, owner, r.Base)
|
||||
}
|
||||
}
|
||||
|
||||
// Standing says where the checkout stands.
|
||||
func (r *Records) Standing() Standing {
|
||||
s := Standing{Repository: r.Repository, Origin: r.Origin}
|
||||
if _, err := os.Stat(filepath.Join(r.Dir(), ".git")); err == nil {
|
||||
s.Commit, _ = git("-C", r.Dir(), "rev-parse", "HEAD")
|
||||
s.Committed, _ = git("-C", r.Dir(), "log", "-1", "--format=%cI")
|
||||
}
|
||||
if s.Commit != "" {
|
||||
s.Documents = len(r.documents())
|
||||
}
|
||||
r.mu.Lock()
|
||||
s.Fetched, s.LastError, s.LeftBehind = r.fetched, r.lastError, r.leftBehind
|
||||
r.mu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
// documents is every markdown document, relative to the root, in a stable order.
|
||||
func (r *Records) documents() []string {
|
||||
var out []string
|
||||
root := r.Dir()
|
||||
_ = filepath.WalkDir(root, func(full string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.Type().IsRegular() && strings.HasSuffix(d.Name(), ".md") {
|
||||
rel, _ := filepath.Rel(root, full)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
emphasis = regexp.MustCompile("[*_`]")
|
||||
spaces = regexp.MustCompile(`\s+`)
|
||||
heading = regexp.MustCompile(`^#{1,6}\s`)
|
||||
hashes = regexp.MustCompile(`^#+\s*`)
|
||||
)
|
||||
|
||||
// flat is a line as a phrase is matched against it: no emphasis marks, one space, lower case.
|
||||
func flat(line string) string {
|
||||
return strings.TrimSpace(spaces.ReplaceAllString(strings.ToLower(emphasis.ReplaceAllString(line, "")), " "))
|
||||
}
|
||||
|
||||
// SearchAnswer is what a search answers.
|
||||
type SearchAnswer struct {
|
||||
Hits []Hit `json:"hits"`
|
||||
More bool `json:"more"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
// Search finds where a phrase appears, case-insensitively, as written — no stemming, no ranking, because a
|
||||
// design record is found by its own words. Bounded, and says when it was.
|
||||
//
|
||||
// **The record is wrapped prose, and a phrase does not know where the line ends.** A line is matched
|
||||
// together with the one after it, joined by a space, and emphasis marks are ignored — `**reachable**` is
|
||||
// the word reachable. A hit names the line it starts on.
|
||||
func (r *Records) Search(query string, limit int) (SearchAnswer, error) {
|
||||
needle := flat(query)
|
||||
if needle == "" {
|
||||
return SearchAnswer{}, errors.New("search for a phrase; an empty one matches every line of every document")
|
||||
}
|
||||
if limit <= 0 || limit > MostHits {
|
||||
limit = MostHits
|
||||
}
|
||||
answer := SearchAnswer{Hits: []Hit{}}
|
||||
for _, p := range r.documents() {
|
||||
raw, err := os.ReadFile(filepath.Join(r.Dir(), filepath.FromSlash(p)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
lines := strings.Split(string(raw), "\n")
|
||||
flats := make([]string, len(lines))
|
||||
for i, l := range lines {
|
||||
flats[i] = flat(l)
|
||||
}
|
||||
head := ""
|
||||
for i, line := range lines {
|
||||
if heading.MatchString(line) {
|
||||
head = strings.TrimSpace(hashes.ReplaceAllString(line, ""))
|
||||
}
|
||||
next := ""
|
||||
if i+1 < len(flats) {
|
||||
next = flats[i+1]
|
||||
}
|
||||
// On this line, or across the break into the next — but not a phrase that begins on the next
|
||||
// line alone, which is that line's hit.
|
||||
onThis := strings.Contains(flats[i], needle)
|
||||
across := !onThis && next != "" && strings.Contains(flats[i]+" "+next, needle) && !strings.Contains(next, needle)
|
||||
if !onThis && !across {
|
||||
continue
|
||||
}
|
||||
if len(answer.Hits) >= limit {
|
||||
answer.More = true
|
||||
break
|
||||
}
|
||||
text := strings.TrimSpace(line)
|
||||
if across {
|
||||
text += " " + strings.TrimSpace(lines[i+1])
|
||||
}
|
||||
answer.Hits = append(answer.Hits, Hit{Path: p, Line: i + 1, Heading: head, Text: text})
|
||||
}
|
||||
if answer.More {
|
||||
break
|
||||
}
|
||||
}
|
||||
answer.Commit = r.Standing().Commit
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// inside cleans a path given relative to the repository, refusing one that leaves it.
|
||||
func inside(p string) (string, bool) {
|
||||
clean := path.Clean(strings.ReplaceAll(p, "\\", "/"))
|
||||
if clean == "." {
|
||||
return "", true
|
||||
}
|
||||
if path.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, "../") {
|
||||
return "", false
|
||||
}
|
||||
return clean, true
|
||||
}
|
||||
|
||||
// Document is one document, as read answers it.
|
||||
type Document struct {
|
||||
Path string `json:"path"`
|
||||
Content string `json:"content"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
// Read is one document, whole, or its first part with a note when it is very long. The path stays inside
|
||||
// the checkout: `..` and absolute paths are refused, not resolved.
|
||||
func (r *Records) Read(p string) (Document, error) {
|
||||
clean, ok := inside(p)
|
||||
if !ok || clean == "" {
|
||||
return Document{}, fmt.Errorf("%q is not a path inside the repository", p)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(r.Dir(), filepath.FromSlash(clean)))
|
||||
if err != nil {
|
||||
return Document{}, fmt.Errorf("the repository holds no %s — `records_list` says what it holds", clean)
|
||||
}
|
||||
d := Document{Path: clean, Content: string(raw), Commit: r.Standing().Commit}
|
||||
if len(d.Content) > MostBytes {
|
||||
d.Content = d.Content[:MostBytes] + "\n\n[… truncated; the document is longer than this answer carries]"
|
||||
d.Truncated = true
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Folder is what a folder holds, one level.
|
||||
type Folder struct {
|
||||
Folder string `json:"folder"`
|
||||
Folders []string `json:"folders"`
|
||||
Documents []string `json:"documents"`
|
||||
}
|
||||
|
||||
// List says what a folder holds: its sub-folders and its documents.
|
||||
func (r *Records) List(folder string) (Folder, error) {
|
||||
clean, ok := inside(folder)
|
||||
if !ok {
|
||||
return Folder{}, fmt.Errorf("%q is not a folder inside the repository", folder)
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(r.Dir(), filepath.FromSlash(clean)))
|
||||
if err != nil {
|
||||
if clean == "" {
|
||||
clean = "/"
|
||||
}
|
||||
return Folder{}, fmt.Errorf("the repository holds no folder %s", clean)
|
||||
}
|
||||
out := Folder{Folder: clean, Folders: []string{}, Documents: []string{}}
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case e.IsDir() && e.Name() != ".git" && e.Name() != "node_modules":
|
||||
out.Folders = append(out.Folders, e.Name())
|
||||
case e.Type().IsRegular() && strings.HasSuffix(e.Name(), ".md"):
|
||||
out.Documents = append(out.Documents, e.Name())
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var repositoryName = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`)
|
||||
|
||||
// FromEnv is the reader as the mesh configures it: the directory it was given, the forge it was bound to,
|
||||
// and the repository its settings name. It refuses to guess any of the three (novox/hq ADR 0112).
|
||||
func FromEnv(env func(string) string) (*Records, error) {
|
||||
dir := env("MESH_RECORDS_DIR")
|
||||
if dir == "" {
|
||||
return nil, errors.New("MESH_RECORDS_DIR is unset: the mesh gives this module the directory its checkout lives in")
|
||||
}
|
||||
originFile := env("MESH_RECORDS_ORIGIN_FILE")
|
||||
if originFile == "" {
|
||||
return nil, errors.New("MESH_RECORDS_ORIGIN_FILE is unset: the forge's address comes from the git provision's binding")
|
||||
}
|
||||
raw, err := os.ReadFile(originFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
origin := strings.TrimSpace(string(raw))
|
||||
if origin == "" {
|
||||
return nil, fmt.Errorf("%s is empty: the git provision has not been bound yet", originFile)
|
||||
}
|
||||
configFile := env("MESH_RECORDS_CONFIG_FILE")
|
||||
if configFile == "" {
|
||||
return nil, errors.New("MESH_RECORDS_CONFIG_FILE is unset")
|
||||
}
|
||||
var config struct {
|
||||
Repository any `json:"repository"`
|
||||
}
|
||||
if err := readJSON(configFile, &config); err != nil {
|
||||
return nil, fmt.Errorf("%s is not JSON: %w", configFile, err)
|
||||
}
|
||||
repository, _ := config.Repository.(string)
|
||||
repository = strings.TrimSpace(repository)
|
||||
if !repositoryName.MatchString(repository) {
|
||||
return nil, errors.New("this module reads the repository its settings name, and none is set: " +
|
||||
"`settings set records <file>` with {\"repository\": \"<owner>/<name>\"} — a module names no mesh (novox/hq ADR 0112)")
|
||||
}
|
||||
return &Records{Base: dir, Origin: origin, Repository: repository}, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
// The reader against a real repository: a checkout, a phrase found where it is written, a document read
|
||||
// whole, a merge pulled — and the check novox/hq ADR 0025 names: search for a phrase that appears only in
|
||||
// one design document, and get it back. Ported with the TypeScript module's tests, and the layout's own.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func run(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %s", args, out)
|
||||
}
|
||||
}
|
||||
|
||||
func write(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
_ = os.MkdirAll(filepath.Dir(path), 0o755)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func aRepository(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
run(t, dir, "init", "--quiet", "--initial-branch=main")
|
||||
run(t, dir, "config", "user.email", "t@example.invalid")
|
||||
run(t, dir, "config", "user.name", "t")
|
||||
write(t, filepath.Join(dir, "README.md"), "# A repository\n\nWhat this is.\n")
|
||||
write(t, filepath.Join(dir, "02-DECISIONS/0001-a-decision.md"), "# 1. A decision\n\n## Context\n\nThe context.\n\n## Decision\n\nWe decided the thing.\n")
|
||||
write(t, filepath.Join(dir, "03-DESIGN/07-knowledge.md"), "# Knowledge\n\n## The stores\n\nSilence and success must never look alike.\n\n"+
|
||||
"A phrase that is wrapped at the\ncolumn where every document wraps, and **reachable is not the same as\nsurfacing** under emphasis.\n")
|
||||
run(t, dir, "add", "-A")
|
||||
run(t, dir, "commit", "--quiet", "-m", "first")
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestAPhraseInOneDesignDocumentComesBackFromWhereItIsWritten(t *testing.T) {
|
||||
origin := aRepository(t)
|
||||
r := &Records{Base: t.TempDir(), Origin: "http://forge.invalid:3000", Repository: "novox/hq", URL: origin}
|
||||
r.Sync()
|
||||
found, err := r.Search("never look alike", 0)
|
||||
if err != nil || len(found.Hits) != 1 || found.Hits[0].Path != "03-DESIGN/07-knowledge.md" || found.Hits[0].Heading != "The stores" {
|
||||
t.Fatalf("%+v %v", found, err)
|
||||
}
|
||||
if !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(found.Commit) {
|
||||
t.Fatalf("the answer names no commit: %q", found.Commit)
|
||||
}
|
||||
// Wrapped prose: a phrase across the break is found once, at the line it starts on; emphasis is not
|
||||
// part of the words; a phrase on one line is not also counted from the line before it.
|
||||
for phrase, line := range map[string]int{"wrapped at the column where": 7, "reachable is not the same as surfacing": 8} {
|
||||
got, _ := r.Search(phrase, 0)
|
||||
if len(got.Hits) != 1 || got.Hits[0].Line != line {
|
||||
t.Errorf("%q: %+v", phrase, got.Hits)
|
||||
}
|
||||
}
|
||||
if got, _ := r.Search("under emphasis", 0); len(got.Hits) != 1 {
|
||||
t.Errorf("a phrase on one line was counted %d times", len(got.Hits))
|
||||
}
|
||||
doc, err := r.Read("02-DECISIONS/0001-a-decision.md")
|
||||
if err != nil || !strings.Contains(doc.Content, "We decided the thing") || doc.Truncated {
|
||||
t.Fatalf("%+v %v", doc, err)
|
||||
}
|
||||
root, _ := r.List("")
|
||||
if strings.Join(root.Folders, ",") != "02-DECISIONS,03-DESIGN" || strings.Join(root.Documents, ",") != "README.md" {
|
||||
t.Fatalf("%+v", root)
|
||||
}
|
||||
if s := r.Standing(); s.Documents != 3 || s.LastError != "" {
|
||||
t.Fatalf("%+v", s)
|
||||
}
|
||||
// A merge on the origin, pulled: the checkout follows and names the new commit.
|
||||
write(t, filepath.Join(origin, "02-DECISIONS/0002-another.md"), "# 2. Another\n\nA phrase nobody wrote before.\n")
|
||||
run(t, origin, "add", "-A")
|
||||
run(t, origin, "commit", "--quiet", "-m", "second")
|
||||
r.Sync()
|
||||
after, _ := r.Search("nobody wrote before", 0)
|
||||
if len(after.Hits) != 1 || after.Commit == found.Commit {
|
||||
t.Fatalf("%+v", after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPathOutsideTheRepositoryAndAnEmptySearchAreRefused(t *testing.T) {
|
||||
r := &Records{Base: t.TempDir(), Origin: "http://forge.invalid:3000", Repository: "novox/hq"}
|
||||
for _, p := range []string{"../etc/passwd", "/etc/passwd", "a/../../b"} {
|
||||
if _, err := r.Read(p); err == nil || !strings.Contains(err.Error(), "not a path inside") {
|
||||
t.Errorf("%s: %v", p, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.Search(" ", 0); err == nil || !strings.Contains(err.Error(), "empty one") {
|
||||
t.Errorf("an empty search: %v", err)
|
||||
}
|
||||
if r.CloneURL() != "http://forge.invalid:3000/novox/hq.git" {
|
||||
t.Error(r.CloneURL())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedSyncLeavesTheCheckoutStandingAndSaysWhy(t *testing.T) {
|
||||
r := &Records{Base: t.TempDir(), Origin: "http://127.0.0.1:1", Repository: "novox/hq"}
|
||||
r.Sync()
|
||||
if s := r.Standing(); s.Commit != "" || s.LastError == "" {
|
||||
t.Fatalf("%+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// The old layout — a checkout in the given directory itself — is cleared where it is this module's, and
|
||||
// the checkout is made in its own directory (novox/hq issue 251).
|
||||
func TestTheOldLayoutIsClearedAndTheCheckoutLivesInItsOwnDirectory(t *testing.T) {
|
||||
origin := aRepository(t)
|
||||
base := t.TempDir()
|
||||
write(t, filepath.Join(base, "README.md"), "an old checkout's file")
|
||||
_ = os.MkdirAll(filepath.Join(base, ".git"), 0o755)
|
||||
r := &Records{Base: base, Origin: "http://forge.invalid:3000", Repository: "novox/hq", URL: origin}
|
||||
r.Sync()
|
||||
if _, err := os.Stat(filepath.Join(base, "README.md")); err == nil {
|
||||
t.Error("the old layout's file was left")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(base, "repository", ".git")); err != nil {
|
||||
t.Fatal("the checkout is not in its own directory")
|
||||
}
|
||||
if s := r.Standing(); s.LeftBehind != "" || s.Documents != 3 {
|
||||
t.Fatalf("%+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheReaderRefusesToGuess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, filepath.Join(dir, "origin"), "http://forge.invalid:3000\n")
|
||||
write(t, filepath.Join(dir, "config.json"), "{}\n")
|
||||
env := map[string]string{"MESH_RECORDS_DIR": dir, "MESH_RECORDS_ORIGIN_FILE": filepath.Join(dir, "origin"),
|
||||
"MESH_RECORDS_CONFIG_FILE": filepath.Join(dir, "config.json")}
|
||||
if _, err := FromEnv(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "none is set") {
|
||||
t.Fatalf("a module with no repository set guessed one: %v", err)
|
||||
}
|
||||
write(t, filepath.Join(dir, "config.json"), `{"repository": "novox/hq"}`)
|
||||
r, err := FromEnv(func(k string) string { return env[k] })
|
||||
if err != nil || r.Repository != "novox/hq" || r.Origin != "http://forge.invalid:3000" {
|
||||
t.Fatalf("%+v %v", r, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module records
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
@@ -1,34 +0,0 @@
|
||||
// records' consumer: keep the checkout current (novox/hq ADR 0153).
|
||||
//
|
||||
// Synced when the runtime binds the broker, on every merge the forge announces, and on a timer for
|
||||
// the merges it did not hear about — a restart during a merge, a repository the forge does not emit
|
||||
// for. The timer is unhurried: the record changes when thinking changes, not by the minute.
|
||||
import { on } from "@novox/mesh-sdk/events";
|
||||
import { recordsFromEnv, type Records } from "./records.js";
|
||||
|
||||
let records: Records | null = null;
|
||||
try {
|
||||
records = await recordsFromEnv();
|
||||
} catch (err) {
|
||||
console.log(`[records] not reading — ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
if (records) {
|
||||
const reader = records;
|
||||
void reader.sync().then(async () => {
|
||||
const s = await reader.standing();
|
||||
console.log(`[records] ${s.repository} at ${s.commit.slice(0, 8) || "(no commit)"}, ${s.documents} document(s)${s.lastError ? ` — ${s.lastError}` : ""}`);
|
||||
});
|
||||
setInterval(() => void reader.sync(), 10 * 60 * 1000).unref();
|
||||
|
||||
// A merge on the forge into the repository this reads: pull now. The event names the repository
|
||||
// by owner and name (gitea's `pull.merged`); anything else is somebody else's merge.
|
||||
await on<{ owner?: string; repo?: string; base?: string }>("gitea.pull.merged", async (event) => {
|
||||
const merged = `${event.body.owner ?? ""}/${event.body.repo ?? ""}`;
|
||||
if (merged !== reader.repository) return;
|
||||
console.log(`[records] ${merged} merged; syncing`);
|
||||
await reader.sync();
|
||||
});
|
||||
}
|
||||
|
||||
export { records };
|
||||
@@ -2,9 +2,6 @@
|
||||
"module": "records",
|
||||
"version": "1",
|
||||
"slug": "records",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"requires": [
|
||||
"git"
|
||||
],
|
||||
@@ -60,14 +57,12 @@
|
||||
{
|
||||
"name": "code",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
],
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/records",
|
||||
"binary": "records",
|
||||
"loads": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
"records"
|
||||
],
|
||||
"env": {
|
||||
"MESH_RECORDS_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"name": "@novox/module-records",
|
||||
"version": "0.1.0",
|
||||
"description": "records — reads a repository of decisions, designs and issues where it is written, and answers questions about it (novox/hq ADR 0025, ADR 0153).",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc records.ts index.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
|
||||
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.6.0"
|
||||
}
|
||||
}
|
||||
@@ -1,277 +0,0 @@
|
||||
/**
|
||||
* The record, read where it is written (novox/hq ADR 0025, ADR 0153).
|
||||
*
|
||||
* A repository of decisions, designs and issues — markdown, nothing else — cloned from the mesh's own
|
||||
* forge and kept current. **A checkout, not a copy**: the same bytes the repository holds, at a commit
|
||||
* every answer names, refreshed on every merge the forge announces and on a timer besides. Nothing is
|
||||
* transformed, indexed or summarised on the way, so there is nothing that can drift from the source
|
||||
* except by lagging behind it, and the lag is a number in every answer.
|
||||
*
|
||||
* What it answers: a search for a phrase, a document by path, what a folder holds, and where the
|
||||
* checkout stands. The reasoning is in the documents; this only finds them.
|
||||
*/
|
||||
import { execFile } from "node:child_process";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { join, normalize, relative, sep } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
/** One place a phrase was found. */
|
||||
export interface Hit {
|
||||
/** The document, relative to the repository's root. */
|
||||
path: string;
|
||||
/** The line it was found on, from 1. */
|
||||
line: number;
|
||||
/** The nearest heading above it, so a hit reads as where in the document it is. */
|
||||
heading: string;
|
||||
/** The line itself, trimmed. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface Standing {
|
||||
repository: string;
|
||||
origin: string;
|
||||
/** The commit the checkout is at, or empty before the first clone. */
|
||||
commit: string;
|
||||
/** When that commit was made, as the repository says. */
|
||||
committed: string;
|
||||
/** When this reader last brought the checkout up to date. */
|
||||
fetched: string;
|
||||
/** How many markdown documents the checkout holds. */
|
||||
documents: number;
|
||||
/** Why the last sync failed, if it did; the checkout stands where it was. */
|
||||
lastError?: string;
|
||||
}
|
||||
|
||||
/** Search answers at most this many places; a phrase found more often is a phrase to narrow. */
|
||||
export const MOST_HITS = 50;
|
||||
/** A document longer than this is answered in part, and says so. */
|
||||
export const MOST_BYTES = 200_000;
|
||||
|
||||
export class Records {
|
||||
/** Where the checkout lives; the mesh gives the module the directory. */
|
||||
readonly dir: string;
|
||||
/** The forge's address, `scheme://host:port`, from the git provision's binding. */
|
||||
readonly origin: string;
|
||||
/** The repository's path on it, `owner/name`, from this module's settings. */
|
||||
readonly repository: string;
|
||||
|
||||
private syncing: Promise<void> | undefined;
|
||||
private fetched = "";
|
||||
private lastError: string | undefined;
|
||||
|
||||
constructor(dir: string, origin: string, repository: string) {
|
||||
this.dir = dir;
|
||||
this.origin = origin;
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
/** The clone URL: the forge, the repository. Public repositories only; a credential would be a
|
||||
* secret this module has not asked for. */
|
||||
get url(): string {
|
||||
return `${this.origin.replace(/\/$/, "")}/${this.repository}.git`;
|
||||
}
|
||||
|
||||
/** Bring the checkout up to date, cloning it if it does not exist. One at a time: a second call
|
||||
* while one runs joins it rather than racing it. Never throws — a failed sync is recorded in
|
||||
* `standing()` and the checkout stands where it was, which is still an answer. */
|
||||
sync(): Promise<void> {
|
||||
if (!this.syncing) {
|
||||
this.syncing = this.doSync().finally(() => {
|
||||
this.syncing = undefined;
|
||||
});
|
||||
}
|
||||
return this.syncing;
|
||||
}
|
||||
|
||||
private async doSync(): Promise<void> {
|
||||
try {
|
||||
const cloned = await exists(join(this.dir, ".git"));
|
||||
if (!cloned) {
|
||||
await fs.mkdir(this.dir, { recursive: true });
|
||||
await run("git", ["clone", "--quiet", "--depth", "50", this.url, this.dir]);
|
||||
} else {
|
||||
// The checkout is the mesh's, so a local change is nobody's: reset to what the forge has,
|
||||
// rather than merging into something a hand may have touched.
|
||||
await run("git", ["-C", this.dir, "fetch", "--quiet", "--depth", "50", "origin"]);
|
||||
await run("git", ["-C", this.dir, "reset", "--quiet", "--hard", "origin/HEAD"]);
|
||||
}
|
||||
this.fetched = new Date().toISOString();
|
||||
this.lastError = undefined;
|
||||
} catch (e) {
|
||||
this.lastError = e instanceof Error ? e.message : String(e);
|
||||
console.error(`[records] could not sync ${this.url}: ${this.lastError}`);
|
||||
}
|
||||
}
|
||||
|
||||
async standing(): Promise<Standing> {
|
||||
let commit = "";
|
||||
let committed = "";
|
||||
if (await exists(join(this.dir, ".git"))) {
|
||||
try {
|
||||
commit = (await run("git", ["-C", this.dir, "rev-parse", "HEAD"])).stdout.trim();
|
||||
committed = (await run("git", ["-C", this.dir, "log", "-1", "--format=%cI"])).stdout.trim();
|
||||
} catch {
|
||||
// A checkout without a commit yet: said as empty rather than thrown.
|
||||
}
|
||||
}
|
||||
const documents = commit ? (await this.documents()).length : 0;
|
||||
return {
|
||||
repository: this.repository,
|
||||
origin: this.origin,
|
||||
commit,
|
||||
committed,
|
||||
fetched: this.fetched,
|
||||
documents,
|
||||
...(this.lastError ? { lastError: this.lastError } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Every markdown document, relative to the root, in a stable order. */
|
||||
async documents(): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const walk = async (at: string): Promise<void> => {
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(at, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (e.name === ".git" || e.name === "node_modules") continue;
|
||||
const full = join(at, e.name);
|
||||
if (e.isDirectory()) await walk(full);
|
||||
else if (e.isFile() && e.name.endsWith(".md")) out.push(relative(this.dir, full).split(sep).join("/"));
|
||||
}
|
||||
};
|
||||
await walk(this.dir);
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a phrase appears, case-insensitively, as written — no stemming, no ranking, because a
|
||||
* design record is found by its own words and a reader deciding which words matter would be a
|
||||
* second opinion about somebody else's document. Bounded, and says when it was.
|
||||
*
|
||||
* **The record is wrapped prose, and a phrase does not know where the line ends.** Every document
|
||||
* here wraps at a hundred columns, so a phrase of six words is as likely to straddle a line break
|
||||
* as not; matched line by line, the first live search for a sentence of ADR 0025 found nothing.
|
||||
* So a line is matched together with the one after it, joined by a space, and emphasis marks
|
||||
* are ignored — `**reachable**` is the word reachable. A hit still names the line it starts on.
|
||||
*/
|
||||
async search(query: string, limit = MOST_HITS): Promise<{ hits: Hit[]; more: boolean; commit: string }> {
|
||||
const needle = plain(query).toLowerCase().replace(/\s+/g, " ").trim();
|
||||
if (!needle) throw new Error("search for a phrase; an empty one matches every line of every document");
|
||||
const cap = Math.max(1, Math.min(limit, MOST_HITS));
|
||||
const hits: Hit[] = [];
|
||||
let more = false;
|
||||
for (const path of await this.documents()) {
|
||||
const text = await fs.readFile(join(this.dir, path), "utf8");
|
||||
let heading = "";
|
||||
const lines = text.split("\n");
|
||||
const flat = lines.map((l) => plain(l).toLowerCase().replace(/\s+/g, " ").trim());
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]!;
|
||||
if (/^#{1,6}\s/.test(line)) heading = line.replace(/^#+\s*/, "").trim();
|
||||
const here = flat[i]!;
|
||||
const next = i + 1 < flat.length ? flat[i + 1]! : "";
|
||||
// On this line, or across the break into the next — but not a phrase that begins on the
|
||||
// next line alone, which is that line's hit.
|
||||
const onThis = here.includes(needle);
|
||||
const acrossTheBreak = !onThis && next !== "" && `${here} ${next}`.includes(needle) && !next.includes(needle);
|
||||
if (onThis || acrossTheBreak) {
|
||||
if (hits.length >= cap) {
|
||||
more = true;
|
||||
break;
|
||||
}
|
||||
hits.push({ path, line: i + 1, heading, text: acrossTheBreak ? `${line.trim()} ${lines[i + 1]!.trim()}` : line.trim() });
|
||||
}
|
||||
}
|
||||
if (more) break;
|
||||
}
|
||||
const commit = (await this.standing()).commit;
|
||||
return { hits, more, commit };
|
||||
}
|
||||
|
||||
/** One document, whole, or its first part with a note when it is very long. The path is kept
|
||||
* inside the checkout: `..` and absolute paths are refused, not resolved. */
|
||||
async read(path: string): Promise<{ path: string; content: string; truncated: boolean; commit: string }> {
|
||||
const clean = normalize(path).split(sep).join("/");
|
||||
if (!clean || clean.startsWith("..") || clean.startsWith("/") || clean.includes("/../")) {
|
||||
throw new Error(`"${path}" is not a path inside the repository`);
|
||||
}
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(join(this.dir, clean), "utf8");
|
||||
} catch {
|
||||
throw new Error(`the repository holds no ${clean} — \`records_list\` says what it holds`);
|
||||
}
|
||||
const truncated = content.length > MOST_BYTES;
|
||||
return {
|
||||
path: clean,
|
||||
content: truncated ? content.slice(0, MOST_BYTES) + "\n\n[… truncated; the document is longer than this answer carries]" : content,
|
||||
truncated,
|
||||
commit: (await this.standing()).commit,
|
||||
};
|
||||
}
|
||||
|
||||
/** What a folder holds: its sub-folders and its documents, one level. */
|
||||
async list(folder = ""): Promise<{ folder: string; folders: string[]; documents: string[] }> {
|
||||
const clean = normalize(folder || ".").split(sep).join("/").replace(/^\.\/?/, "");
|
||||
if (clean.startsWith("..") || clean.startsWith("/")) {
|
||||
throw new Error(`"${folder}" is not a folder inside the repository`);
|
||||
}
|
||||
const at = clean ? join(this.dir, clean) : this.dir;
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(at, { withFileTypes: true });
|
||||
} catch {
|
||||
throw new Error(`the repository holds no folder ${clean || "/"}`);
|
||||
}
|
||||
const folders = entries.filter((e) => e.isDirectory() && e.name !== ".git" && e.name !== "node_modules").map((e) => e.name).sort();
|
||||
const documents = entries.filter((e) => e.isFile() && e.name.endsWith(".md")).map((e) => e.name).sort();
|
||||
return { folder: clean, folders, documents };
|
||||
}
|
||||
}
|
||||
|
||||
/** The reader as the mesh configures it: the directory it was given, the forge it was bound to, and
|
||||
* the repository its settings name. Refuses to guess any of the three (novox/hq ADR 0112). */
|
||||
export async function recordsFromEnv(env: NodeJS.ProcessEnv = process.env): Promise<Records> {
|
||||
const dir = env.MESH_RECORDS_DIR;
|
||||
if (!dir) throw new Error("MESH_RECORDS_DIR is unset: the mesh gives this module the directory its checkout lives in");
|
||||
const originFile = env.MESH_RECORDS_ORIGIN_FILE;
|
||||
if (!originFile) throw new Error("MESH_RECORDS_ORIGIN_FILE is unset: the forge's address comes from the git provision's binding");
|
||||
const origin = (await fs.readFile(originFile, "utf8")).trim();
|
||||
if (!origin) throw new Error(`${originFile} is empty: the git provision has not been bound yet`);
|
||||
const configFile = env.MESH_RECORDS_CONFIG_FILE;
|
||||
if (!configFile) throw new Error("MESH_RECORDS_CONFIG_FILE is unset");
|
||||
let config: { repository?: unknown } = {};
|
||||
try {
|
||||
config = JSON.parse(await fs.readFile(configFile, "utf8")) as { repository?: unknown };
|
||||
} catch (e) {
|
||||
throw new Error(`${configFile} is not JSON: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
const repository = typeof config.repository === "string" ? config.repository.trim() : "";
|
||||
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) {
|
||||
throw new Error(
|
||||
"this module reads the repository its settings name, and none is set: " +
|
||||
'`settings set records <file>` with {"repository": "<owner>/<name>"} — a module names no mesh (novox/hq ADR 0112)',
|
||||
);
|
||||
}
|
||||
return new Records(dir, origin, repository);
|
||||
}
|
||||
|
||||
/** A line without its markdown emphasis, so a phrase matches the words and not the marks. */
|
||||
function plain(line: string): string {
|
||||
return line.replace(/[*_`]/g, "");
|
||||
}
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
/**
|
||||
* The reader against a real repository: a checkout, a phrase found where it is written, a document
|
||||
* read whole, a merge pulled — and the check novox/hq ADR 0025 names: search for a phrase that appears
|
||||
* only in one design document, and get it back.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { Records } from "../records.ts";
|
||||
|
||||
function aRepository(): string {
|
||||
const dir = mkdtempSync("/tmp/records-origin-");
|
||||
const git = (...args: string[]) => execFileSync("git", ["-C", dir, ...args], { stdio: "pipe" });
|
||||
git("init", "--quiet", "--initial-branch=main");
|
||||
git("config", "user.email", "t@example.invalid");
|
||||
git("config", "user.name", "t");
|
||||
mkdirSync(join(dir, "02-DECISIONS"));
|
||||
mkdirSync(join(dir, "03-DESIGN"));
|
||||
writeFileSync(join(dir, "README.md"), "# A repository\n\nWhat this is.\n");
|
||||
writeFileSync(join(dir, "02-DECISIONS/0001-a-decision.md"), "# 1. A decision\n\n## Context\n\nThe context.\n\n## Decision\n\nWe decided the thing.\n");
|
||||
writeFileSync(join(dir, "03-DESIGN/07-knowledge.md"), "# Knowledge\n\n## The stores\n\nSilence and success must never look alike.\n\nA phrase that is wrapped at the\ncolumn where every document wraps, and **reachable is not the same as\nsurfacing** under emphasis.\n");
|
||||
git("add", "-A");
|
||||
git("commit", "--quiet", "-m", "first");
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("a phrase that appears in one design document comes back from where it is written", async () => {
|
||||
const origin = aRepository();
|
||||
const records = new Records(mkdtempSync("/tmp/records-checkout-"), "file://" + origin.replace(/\/[^/]+$/, ""), origin.split("/").pop()!);
|
||||
// A file:// origin has no `.git` suffix; point the clone at the directory itself.
|
||||
Object.defineProperty(records, "url", { get: () => origin });
|
||||
await records.sync();
|
||||
const found = await records.search("never look alike");
|
||||
assert.equal(found.hits.length, 1);
|
||||
assert.equal(found.hits[0]!.path, "03-DESIGN/07-knowledge.md");
|
||||
assert.equal(found.hits[0]!.heading, "The stores");
|
||||
assert.match(found.commit, /^[0-9a-f]{40}$/, "the answer names the commit it was read at");
|
||||
|
||||
// Wrapped prose: a phrase across the line break is found, once, at the line it starts on; and
|
||||
// emphasis marks are not part of the words.
|
||||
const wrapped = await records.search("wrapped at the column where");
|
||||
assert.deepEqual(wrapped.hits.map((h) => [h.path, h.line]), [["03-DESIGN/07-knowledge.md", 7]]);
|
||||
const emphasised = await records.search("reachable is not the same as surfacing");
|
||||
assert.deepEqual(emphasised.hits.map((h) => [h.path, h.line]), [["03-DESIGN/07-knowledge.md", 8]]);
|
||||
const alsoOnOneLine = await records.search("under emphasis");
|
||||
assert.equal(alsoOnOneLine.hits.length, 1, "a phrase on one line is not also counted from the line before it");
|
||||
|
||||
const doc = await records.read("02-DECISIONS/0001-a-decision.md");
|
||||
assert.match(doc.content, /We decided the thing/);
|
||||
assert.equal(doc.truncated, false);
|
||||
|
||||
const root = await records.list();
|
||||
assert.deepEqual(root.folders, ["02-DECISIONS", "03-DESIGN"]);
|
||||
assert.deepEqual(root.documents, ["README.md"]);
|
||||
|
||||
const standing = await records.standing();
|
||||
assert.equal(standing.documents, 3);
|
||||
assert.equal(standing.lastError, undefined);
|
||||
|
||||
// A merge on the origin, pulled: the checkout follows the source and names the new commit.
|
||||
writeFileSync(join(origin, "02-DECISIONS/0002-another.md"), "# 2. Another\n\nA phrase nobody wrote before.\n");
|
||||
execFileSync("git", ["-C", origin, "add", "-A"], { stdio: "pipe" });
|
||||
execFileSync("git", ["-C", origin, "commit", "--quiet", "-m", "second"], { stdio: "pipe" });
|
||||
await records.sync();
|
||||
const after = await records.search("nobody wrote before");
|
||||
assert.equal(after.hits.length, 1);
|
||||
assert.notEqual(after.commit, found.commit);
|
||||
});
|
||||
|
||||
test("a path outside the repository is refused, and an empty search is too", async () => {
|
||||
const records = new Records(mkdtempSync("/tmp/records-checkout-"), "http://forge.invalid:3000", "novox/hq");
|
||||
await assert.rejects(() => records.read("../etc/passwd"), /not a path inside/);
|
||||
await assert.rejects(() => records.read("/etc/passwd"), /not a path inside/);
|
||||
await assert.rejects(() => records.search(" "), /empty one/);
|
||||
assert.equal(records.url, "http://forge.invalid:3000/novox/hq.git");
|
||||
});
|
||||
|
||||
test("a sync that fails leaves the checkout standing and says why", async () => {
|
||||
const records = new Records(mkdtempSync("/tmp/records-checkout-"), "http://127.0.0.1:1", "novox/hq");
|
||||
await records.sync();
|
||||
const standing = await records.standing();
|
||||
assert.equal(standing.commit, "");
|
||||
assert.ok(standing.lastError, "a failed sync is said, not swallowed");
|
||||
});
|
||||
@@ -1,62 +0,0 @@
|
||||
// records' tools — how the record is asked (novox/hq ADR 0025, ADR 0153).
|
||||
//
|
||||
// Five questions, each answered from the checkout at the commit it names: where does a phrase
|
||||
// appear, what does one document say, what does a folder hold, where does the checkout stand, and
|
||||
// bring it up to date now. The reasoning stays in the documents; the tools only find them.
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { recordsFromEnv, type Records } from "../records.js";
|
||||
|
||||
export function getRecordsTools(records: Records): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "records_search",
|
||||
description:
|
||||
"Where a phrase appears in the decisions, designs and issues, as written: document, line, nearest heading. " +
|
||||
"Search the literal words of a symptom or a term before forming a hypothesis; the answer names the commit it was read at.",
|
||||
input: {
|
||||
query: { type: "string", description: "the phrase, matched case-insensitively as written" },
|
||||
limit: { type: "number", description: "at most this many places (default 50)" },
|
||||
},
|
||||
run: async (args) => records.search(String(args.query ?? ""), args.limit ? Number(args.limit) : undefined),
|
||||
},
|
||||
{
|
||||
name: "records_read",
|
||||
description: "One document, whole, by its path in the repository — a decision record, a design document, an issue report.",
|
||||
input: { path: { type: "string", description: "the document's path, e.g. 02-DECISIONS/0025-....md" } },
|
||||
run: async (args) => records.read(String(args.path ?? "")),
|
||||
},
|
||||
{
|
||||
name: "records_list",
|
||||
description: "What a folder of the repository holds: its sub-folders and its documents. The root when no folder is named.",
|
||||
input: { folder: { type: "string", description: "a folder inside the repository (optional)" } },
|
||||
run: async (args) => records.list(args.folder ? String(args.folder) : ""),
|
||||
},
|
||||
{
|
||||
name: "records_status",
|
||||
description: "Where the checkout stands: the repository, the forge it is read from, the commit and its date, when it was last brought up to date.",
|
||||
input: {},
|
||||
run: async () => records.standing(),
|
||||
},
|
||||
{
|
||||
name: "records_sync",
|
||||
description: "Bring the checkout up to date now, and say where it stands.",
|
||||
input: {},
|
||||
run: async () => {
|
||||
await records.sync();
|
||||
return records.standing();
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// The reader is made once, at load, from the environment the runtime resolves; the contributor is
|
||||
// synchronous and is called at every collection. Without a repository to read there is nothing to
|
||||
// answer, and the module exposes no tools rather than five that fail — the sdk's contract: a
|
||||
// contributor returning [] is normal.
|
||||
let reader: Records | null = null;
|
||||
try {
|
||||
reader = await recordsFromEnv();
|
||||
} catch (err) {
|
||||
console.log(`[records] no tools — ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
registerModuleTools("records", () => (reader ? getRecordsTools(reader) : []));
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["records.ts", "index.ts", "tools/index.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user