package main import ( "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "regexp" "sort" "strings" "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" ) // The import of slice 3 (novox/hq ADR 0275 §8, to-be 53 §8): every issue folder in hq, read through the // records module over the bus, written as an issue under its own number. It only ever creates an issue // that is not there, or fills the status of one the tracker holds without a status (opened with a folder // before this slice); it never overwrites, never advances the counter, and run twice changes nothing and // says so. A dry run reads everything, writes nothing, and says what it would do. // Records is the record as the records module serves it (to-be 53 §8: "read through the records // module"): where its checkout stands, what a folder holds, and one document whole. type Records interface { // Standing is the commit the checkout is at. Standing() (string, error) // List is a folder's sub-folders and documents. List(folder string) (folders, documents []string, err error) // Read is one document and the commit it was read at. Read(path string) (content, commit string, err error) } // busRecords is Records over the bus: the records module's tools, called as this module (SDK Ask). type busRecords struct{} func (busRecords) Standing() (string, error) { raw, err := stdio.Ask("records.records_status", map[string]any{}) if err != nil { return "", err } var s struct { Commit string `json:"commit"` LastError string `json:"lastError"` } if err := json.Unmarshal(raw, &s); err != nil { return "", fmt.Errorf("records_status answered something else: %v", err) } if s.Commit == "" { return "", fmt.Errorf("the record's checkout stands at no commit yet (%s)", s.LastError) } return s.Commit, nil } func (busRecords) List(folder string) ([]string, []string, error) { raw, err := stdio.Ask("records.records_list", map[string]any{"folder": folder}) if err != nil { return nil, nil, err } var f struct { Folders []string `json:"folders"` Documents []string `json:"documents"` } if err := json.Unmarshal(raw, &f); err != nil { return nil, nil, fmt.Errorf("records_list answered something else: %v", err) } return f.Folders, f.Documents, nil } func (busRecords) Read(path string) (string, string, error) { raw, err := stdio.Ask("records.records_read", map[string]any{"path": path}) if err != nil { return "", "", err } var d struct { Content string `json:"content"` Commit string `json:"commit"` Truncated bool `json:"truncated"` } if err := json.Unmarshal(raw, &d); err != nil { return "", "", fmt.Errorf("records_read answered something else: %v", err) } if d.Truncated { return "", "", fmt.Errorf("%s is longer than the records module answers whole", path) } return d.Content, d.Commit, nil } // dirRecords is Records over a checkout on disk: the terminal's dry run before the module is rolled out, // and the tests. Its commit is `dir:` and the directory, as no forge stands behind it. type dirRecords struct{ dir string } func (d dirRecords) Standing() (string, error) { if st, err := os.Stat(d.dir); err != nil || !st.IsDir() { return "", fmt.Errorf("%s is not a checkout", d.dir) } if out, err := os.ReadFile(filepath.Join(d.dir, ".git", "HEAD")); err == nil { ref := strings.TrimSpace(string(out)) if strings.HasPrefix(ref, "ref: ") { if sha, err := os.ReadFile(filepath.Join(d.dir, ".git", strings.TrimPrefix(ref, "ref: "))); err == nil { return strings.TrimSpace(string(sha)), nil } } else if ref != "" { return ref, nil } } return "dir:" + d.dir, nil } func (d dirRecords) List(folder string) ([]string, []string, error) { entries, err := os.ReadDir(filepath.Join(d.dir, filepath.FromSlash(folder))) if err != nil { return nil, nil, err } var folders, docs []string for _, e := range entries { switch { case e.IsDir(): folders = append(folders, e.Name()) case strings.HasSuffix(e.Name(), ".md"): docs = append(docs, e.Name()) } } return folders, docs, nil } func (d dirRecords) Read(path string) (string, string, error) { raw, err := os.ReadFile(filepath.Join(d.dir, filepath.FromSlash(path))) if err != nil { return "", "", err } commit, _ := d.Standing() return string(raw), commit, nil } // IssuesFolder is where hq keeps its issue folders. const IssuesFolder = "04-ISSUES" // readers is how many documents are read from the record at once. const readers = 8 // commitPrefix is what the commit argument may be: a commit's id, or a prefix of at least seven digits. var commitPrefix = regexp.MustCompile(`^[0-9a-f]{7,40}$`) // errRecordMoved is the record's checkout moving to another commit while the import read it: the folders // read would be of two commits, so nothing is written. var errRecordMoved = errors.New("the record moved while it was read") // EventImported is the event said for each issue the import wrote or filled. const EventImported = "imported" // ImportAsk is what an import carries. type ImportAsk struct { // Commit, when given, is the commit of hq the record must stand at; the import refuses another. Commit string // DryRun reads everything, writes nothing, and says what it would do. DryRun bool } // The actions an import takes on one folder. const ( ActionCreate = "create" // the tracker holds no issue under the number: written ActionFill = "fill" // the tracker holds the issue without a status: its status and fields taken ActionSame = "same" // the tracker holds the issue as the folder has it: nothing to do ActionDiffers = "differs" // the tracker holds the issue differently: the tracker is the truth, nothing done ActionRefused = "refused" // the folder could not be imported, and why ActionWould = "would-" // prefixed to create and fill in a dry run ) // ImportRow is what the import did, or would do, with one folder. type ImportRow struct { Number uint64 `json:"number"` Action string `json:"action"` Status string `json:"status,omitempty"` Title string `json:"title,omitempty"` Folder string `json:"folder"` Why string `json:"why,omitempty"` } // Imported is the answer to an import. type Imported struct { DryRun bool `json:"dry-run"` Commit string `json:"commit"` Folders int `json:"folders"` Created int `json:"created"` Filled int `json:"filled"` Same int `json:"same"` Differs int `json:"differs"` Refused int `json:"refused"` ByStatus map[string]int `json:"by-status"` Rows []ImportRow `json:"rows"` Odd []string `json:"odd,omitempty"` // UnverifiedOwners is each entry of located-in that named no registered owner, with the issues it is in. UnverifiedOwners map[string][]uint64 `json:"unverified-owners,omitempty"` Announced int `json:"announced"` Said string `json:"said"` Warning string `json:"warning,omitempty"` } // candidate is one folder read and checked, before the tracker is asked what it holds. type candidate struct { folder string number uint64 report *Report issue *Issue // unverified is the entries of located-in that named no registered owner. unverified []string err error } // Import reads every issue folder of hq through the record and writes each as an issue under its own // number: create-if-absent on the key, the counter untouched. See the file's comment for what it never // does. func (t *Tracker) Import(a ImportAsk, records Records) (*Imported, error) { return t.importAs(t.by(), a, records) } // importAs is Import as the one who acts. func (t *Tracker) importAs(by string, a ImportAsk, records Records) (*Imported, error) { settings, err := t.Settings() if err != nil { return nil, fmt.Errorf("the settings cannot be read (%v); nothing was imported", err) } if settings.FirstNumber < 1 { return nil, errors.New("the setting first-number is not set: it is the boundary between the folders numbered " + "before the tracker and the issues the tracker numbered itself. Nothing was imported") } owners := Owners(settings.Owners) commit, err := records.Standing() if err != nil { return nil, fmt.Errorf("the record cannot be read (%v); nothing was imported", err) } want := strings.TrimSpace(a.Commit) if want != "" && !commitPrefix.MatchString(want) { return nil, errors.New("commit is a commit's id or a prefix of it, 7 to 40 hex digits; nothing was imported") } if want != "" && !strings.HasPrefix(commit, want) { return nil, fmt.Errorf("the record stands at hq commit %s, not %s: bring it up to date (records_sync), or "+ "name the commit it stands at. Nothing was imported", short(commit), want) } folders, _, err := records.List(IssuesFolder) if err != nil { return nil, fmt.Errorf("the record's %s cannot be listed (%v); nothing was imported", IssuesFolder, err) } out := &Imported{DryRun: a.DryRun, Commit: commit, ByStatus: map[string]int{}, Rows: []ImportRow{}} at := t.now() // Every folder's number, and no two folders sharing one. var cands []*candidate byNumber := map[uint64][]string{} for _, f := range folders { n, ok := folderNumber(f) if !ok { out.Odd = append(out.Odd, fmt.Sprintf("%s/%s is not an issue folder (NNN-slug) and was skipped", IssuesFolder, f)) continue } byNumber[n] = append(byNumber[n], f) cands = append(cands, &candidate{folder: f, number: n}) } out.Folders = len(cands) for _, c := range cands { if len(byNumber[c.number]) > 1 { c.err = fmt.Errorf("two folders are numbered %d: %s — a number names one issue", c.number, strings.Join(byNumber[c.number], " and ")) } } // The issues a report may link: the folders read, and the issues the tracker keeps. exists := map[uint64]bool{} for _, c := range cands { exists[c.number] = true } if keys, err := t.Store.Keys(IssuesState); err == nil { for _, k := range keys { if n, ok := numberOfKey(k); ok { exists[n] = true } } } else { return nil, fmt.Errorf("the issues kept cannot be listed (%v); nothing was imported", err) } // Read and check every report, a few at a time. var wg sync.WaitGroup sem := make(chan struct{}, readers) for _, c := range cands { if c.err != nil { continue } wg.Add(1) sem <- struct{}{} go func(c *candidate) { defer wg.Done() defer func() { <-sem }() path := IssuesFolder + "/" + c.folder + "/00-report.md" text, readAt, err := records.Read(path) if err != nil { c.err = fmt.Errorf("%s could not be read: %v", path, err) return } if readAt != commit { c.err = fmt.Errorf("%w: it stood at %s and %s was read at %s. Run the import again", errRecordMoved, short(commit), path, short(readAt)) return } c.report, c.err = ParseReport(c.folder, text) if c.err != nil { return } c.issue, c.unverified, c.err = t.issueOf(c.report, owners, commit, at, by, exists) }(c) } wg.Wait() for _, c := range cands { if errors.Is(c.err, errRecordMoved) { return nil, fmt.Errorf("%v; nothing was imported", c.err) } } // The counter is never passed: a number the counter has not handed out is the counter's to hand out. var counter uint64 if e, err := t.Store.Get(CountersState, IssueCounter); err != nil { return nil, fmt.Errorf("the issue counter cannot be read (%v); nothing was imported", err) } else if e != nil { if err := json.Unmarshal(e.Value, &counter); err != nil { return nil, fmt.Errorf("the issue counter holds %s, not a number; nothing was imported", e.Value) } for _, c := range cands { if c.err == nil && c.number > counter { return nil, fmt.Errorf("the folder %s is numbered past the counter, which stands at %d: a number the counter "+ "has not handed out is not a folder's to take. Nothing was imported", c.folder, counter) } } } sort.Slice(cands, func(i, j int) bool { return cands[i].number < cands[j].number }) unverified := map[string][]uint64{} for _, c := range cands { row := ImportRow{Number: c.number, Folder: IssuesFolder + "/" + c.folder} if c.report != nil { row.Status, row.Title = c.report.Status, c.report.Title for _, o := range c.report.Odd { out.Odd = append(out.Odd, fmt.Sprintf("%d: %s", c.number, o)) } } if c.err == nil { c.err = t.importOne(c, settings.FirstNumber, a.DryRun, &row, out) } if c.err != nil { row.Action, row.Why = ActionRefused, c.err.Error() } switch row.Action { case ActionRefused: out.Refused++ case ActionSame: out.Same++ case ActionDiffers: out.Differs++ default: if strings.HasSuffix(row.Action, ActionFill) { out.Filled++ } else { out.Created++ } out.ByStatus[row.Status]++ for _, u := range c.unverified { unverified[u] = append(unverified[u], c.number) } } out.Rows = append(out.Rows, row) } if len(unverified) > 0 { out.UnverifiedOwners = unverified } // An issue the tracker holds without a status, whose folder hq does not hold at this commit (opened with // a folder before slice 3, the folder not merged yet): it is open, so no issue is left without a status. folderOf := map[uint64]bool{} for _, c := range cands { folderOf[c.number] = true } keys, err := t.Store.Keys(IssuesState) if err != nil { return nil, fmt.Errorf("the issues kept cannot be listed (%v); what was imported before this is kept", err) } for _, k := range keys { n, ok := numberOfKey(k) if !ok || folderOf[n] { continue } is, _, err := read(t.Store, n) if err != nil || is.Status != "" { continue } row := ImportRow{Number: n, Action: ActionWould + ActionFill, Status: StatusOpen, Title: is.Title, Folder: is.Record, Why: "the tracker holds it without a status and hq holds no folder for it at this commit: open"} if !a.DryRun { row.Action = ActionFill if err := t.fillOpen(n, commit, at, by); err != nil { row.Action, row.Why = ActionRefused, err.Error() out.Refused++ out.Rows = append(out.Rows, row) continue } } out.Filled++ out.ByStatus[StatusOpen]++ out.Rows = append(out.Rows, row) } verb := "created" if a.DryRun { verb = "would create" } out.Said = fmt.Sprintf("Read %d issue folders of hq at commit %s: %s %d, filled %d, the same %d, differing %d, refused %d.", out.Folders, short(commit), verb, out.Created, out.Filled, out.Same, out.Differs, out.Refused) if a.DryRun { out.Said += " Nothing was written: a dry run." } if out.Folders > 0 && out.Created == 0 && out.Filled == 0 && !a.DryRun { if out.Refused == 0 { out.Said += " Nothing changed: every folder is already in the tracker." } else { out.Said += " Nothing changed: every folder not refused is already in the tracker." } } if !a.DryRun && out.Announced < out.Created+out.Filled { out.Warning = fmt.Sprintf("%d of %d changes are kept, and their %s event could not be said on the bus: what watches "+ "the events did not hear of them", out.Created+out.Filled-out.Announced, out.Created+out.Filled, EventImported) } return out, nil } // issueOf is the issue a report becomes: the fields as the folder has them, checked for size and shape as // `open` and `move` check them — but located-in is kept as written even when it names no registered // owner, marked unverified, and no cycle rule is applied: the status is the folder's, as it stood. func (t *Tracker) issueOf(r *Report, owners Owners, commit string, at time.Time, by string, exists map[uint64]bool) (*Issue, []string, error) { title, err := singleLine("title", r.Title, MostTitle) if err != nil { return nil, nil, err } body := r.Body if strings.TrimSpace(body) == "" { body = "(the report says nothing beyond its title)" } report, err := multiLine("report", body, MostReport) if err != nil { return nil, nil, err } var unverified []string // An entry hq wrote is prose at times, longer than one typed at a verb: it is taken whole, within a reason's bound. located, err := entries("located-in", r.LocatedIn, func(e string) (string, error) { v, err := singleLine("an entry of located-in", e, MostReason) if err != nil { return "", err } if owners.owns(v) == "" { unverified = append(unverified, v) } return v, nil }) if err != nil { return nil, nil, err } fixed, err := entries("fixed-by", r.FixedBy, func(e string) (string, error) { return singleLine("an entry of fixed-by", e, MostReason) }) if err != nil { return nil, nil, err } designs, err := entries("amended-design", r.AmendedDesign, func(e string) (string, error) { v, err := singleLine("an entry of amended-design", e, MostReason) if err != nil { return "", err } if !designPath.MatchString(v) { r.Odd = append(r.Odd, fmt.Sprintf("amended-design %q is not a design document's path; kept as written", v)) } return v, nil }) if err != nil { return nil, nil, err } var replay, replayNone string if r.Replay != "" { if replay, err = singleLine("replay", r.Replay, MostLine); err != nil { return nil, nil, err } if !replayID.MatchString(replay) { r.Odd = append(r.Odd, fmt.Sprintf("replay %q names no replay id (R and a number); kept as written", replay)) } } if r.ReplayNone != "" { if replayNone, err = multiLine("replay-none", r.ReplayNone, MostReason); err != nil { return nil, nil, err } } is := &Issue{Number: r.Number, Title: title, Status: r.Status, Report: report, Opened: at, OpenedBy: ImportedBy, LocatedIn: located, LocatedInUnverified: unverified, FixedBy: fixed, AmendedDesign: designs, Replay: replay, ReplayNone: replayNone, Record: r.Folder, Opening: "import@" + short(commit), Imported: &Import{At: at, By: by, Commit: commit, From: r.Path}} if d, err := time.Parse("2006-01-02", r.Opened); err == nil { is.Opened = d } else { r.Odd = append(r.Odd, "no opened date: the import's time is taken") } if d, err := time.Parse("2006-01-02", r.Closed); err == nil { is.Closed = &d } for _, other := range r.Issues { if !exists[other] { r.Odd = append(r.Odd, fmt.Sprintf("names issue %d, which is neither a folder nor an issue kept; not linked", other)) continue } is.Links = append(is.Links, Link{Kind: LinkRelatesTo, Target: keyOf(other), By: ImportedBy, At: at}) } for _, d := range r.Decisions { is.Links = append(is.Links, Link{Kind: LinkDecision, Target: d, By: ImportedBy, At: at}) } if len(is.Links) > MostLinks { is.Links = is.Links[:MostLinks] r.Odd = append(r.Odd, fmt.Sprintf("names more than %d issues and decisions; the first %d are linked", MostLinks, MostLinks)) } raw, err := json.Marshal(is) if err != nil { return nil, nil, err } if len(raw) > MostIssueBytes { return nil, nil, fmt.Errorf("the issue would be %d bytes; one issue is kept in at most %d", len(raw), MostIssueBytes) } return is, unverified, nil } // importOne does what the tracker's state asks for one folder: create, fill, nothing, or refuse. func (t *Tracker) importOne(c *candidate, first uint64, dry bool, row *ImportRow, out *Imported) error { kept, _, err := read(t.Store, c.number) if err != nil && !errors.Is(err, ErrNoIssue) { return err } prefix := "" if dry { prefix = ActionWould } switch { case kept == nil && c.number >= first: return fmt.Errorf("numbered %d, at or past the tracker's first number %d, and the tracker holds no issue %d: "+ "a folder from the tracker's time carries a number the tracker gave, so this one was made without it", c.number, first, c.number) case kept == nil: row.Action = prefix + ActionCreate if dry { return nil } return t.importCreate(c.issue, out) case kept.Status == "" && kept.Record == c.issue.Record: row.Action = prefix + ActionFill if kept.Title != c.issue.Title { out.Odd = append(out.Odd, fmt.Sprintf("%d: the tracker's title %q and the folder's %q differ; the tracker's is kept", c.number, kept.Title, c.issue.Title)) } if dry { return nil } return t.importFill(c.issue, out) case kept.Status == "": row.Action = ActionDiffers row.Why = fmt.Sprintf("the tracker holds issue %d without a status under the folder %s, and this folder is %s: "+ "nothing was changed. If they are one issue, move it in the tracker; its status is not taken from a folder "+ "that is not its own", c.number, kept.Record, c.issue.Record) return nil default: differs := differences(kept, c.issue) if len(differs) == 0 { row.Action = ActionSame return nil } row.Action = ActionDiffers row.Why = "the tracker holds it differently in " + strings.Join(differs, ", ") + "; the tracker is the truth, nothing was changed" return nil } } // differences is the fields in which the tracker's issue and the folder's differ: the fields the import // sets, for an issue the import wrote; only the status and what it needs, for one the tracker opened. func differences(kept, folder *Issue) []string { var d []string same := func(name string, a, b any) { if !reflect.DeepEqual(a, b) { d = append(d, name) } } if kept.OpenedBy == ImportedBy { same("title", kept.Title, folder.Title) same("report", kept.Report, folder.Report) same("record", kept.Record, folder.Record) } same("status", kept.Status, folder.Status) same("located-in", kept.LocatedIn, folder.LocatedIn) same("fixed-by", kept.FixedBy, folder.FixedBy) same("amended-design", kept.AmendedDesign, folder.AmendedDesign) same("replay", kept.Replay, folder.Replay) same("replay-none", kept.ReplayNone, folder.ReplayNone) return d } // importCreate writes a new issue under its number, never over one: its first operation appended to the log at // expect 0 (novox/hq ADR 0300 §1), then its fold to the bucket. func (t *Tracker) importCreate(is *Issue, out *Imported) error { if there, _, err := read(t.Store, is.Number); err == nil && there.Opening != is.Opening { return fmt.Errorf("an issue %d appeared while the import ran; nothing was overwritten. Run the import again", is.Number) } seq, err := t.begin(is, Act{Op: OpImported, By: importer(is)}) if errors.Is(err, errBegun) { return fmt.Errorf("an issue %d appeared while the import ran; nothing was overwritten. Run the import again", is.Number) } if errors.Is(err, errUnsure) { return fmt.Errorf("issue %d may have been written: the log did not confirm it (%v); run the import again, "+ "which finds it the same or creates it", is.Number, err) } if err != nil { return fmt.Errorf("issue %d could not be written: the log did not take it (%v)", is.Number, err) } is.Folded = seq t.store(is.Number, is, 0) t.announceImport(is, out) return nil } // importer is who ran the import that took an issue from hq. func importer(is *Issue) string { if is.Imported != nil && is.Imported.By != "" { return is.Imported.By } return ImportedBy } // importFill takes the status and the fields of an issue the tracker holds without a status: opened with a // folder before slice 3, whose status lived in that folder. Its title, report, opening, notes and links // stay; links the report names are added. func (t *Tracker) importFill(from *Issue, out *Imported) error { is, err := t.change(from.Number, Act{Op: OpImported, By: importer(from)}, func(is *Issue) error { if is.Status != "" { return fmt.Errorf("issue %d took a status while the import ran; nothing was changed", is.Number) } is.Status = from.Status is.Closed = from.Closed is.LocatedIn, is.LocatedInUnverified = from.LocatedIn, from.LocatedInUnverified is.FixedBy, is.AmendedDesign = from.FixedBy, from.AmendedDesign is.Replay, is.ReplayNone = from.Replay, from.ReplayNone is.Imported = from.Imported for _, l := range from.Links { has := false for _, k := range is.Links { if k.Kind == l.Kind && k.Target == l.Target { has = true } } if !has && len(is.Links) < MostLinks { is.Links = append(is.Links, l) } } return nil }) if err != nil { return err } t.announceImport(is, out) return nil } func (t *Tracker) announceImport(is *Issue, out *Imported) { ok, _ := t.emit(EventImported, map[string]any{"number": is.Number, "title": is.Title, "status": is.Status, "record": is.Record, "commit": is.Imported.Commit, "by": is.Imported.By, "at": is.Imported.At}) if ok { out.Announced++ } } // fillOpen gives an issue the tracker holds without a status — opened with a folder before slice 3, whose // folder hq does not hold at the import's commit — the status open, so no issue is left without one. func (t *Tracker) fillOpen(n uint64, commit string, at time.Time, by string) error { is, err := t.change(n, Act{Op: OpImported, By: by}, func(is *Issue) error { if is.Status != "" { return fmt.Errorf("issue %d took a status while the import ran; nothing was changed", is.Number) } is.Status = StatusOpen is.Imported = &Import{At: at, By: by, Commit: commit} return nil }) if err != nil { return err } _, _ = t.emit(EventImported, map[string]any{"number": is.Number, "title": is.Title, "status": is.Status, "record": is.Record, "commit": commit, "by": by, "at": at}) return nil } func short(commit string) string { if len(commit) > 12 && !strings.HasPrefix(commit, "dir:") { return commit[:12] } return commit }