mesh-issues: keep every issue in the log changes and the bucket as its fold, so the record outlives the bucket's cap and a week of events (hq ADR 0300, issues 501, 502)

Every write appends one operation at the issue's last sequence before the
bucket is written; a refused append writes nothing. The start carries what
the bucket holds, refolds what is behind and indexes every issue in memory,
so list answers from memory with a ranked search and a before cursor.

modules/mesh-issues/cmd/mesh-issues/tracker.go

@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
@@ -19,7 +20,7 @@ import (
 // change as an event.
 type Tracker struct {
 	Store Store
-	// Emit says an event as this module (ADR 0041); the events are the audit trail.
+	// Emit says an event as this module (ADR 0041); the events announce, the log is the audit trail (ADR 0300 §4).
 	Emit     func(event string, body any) error
 	Settings func() (Settings, error)
 	Now      func() time.Time
@@ -39,7 +40,11 @@ type Tracker struct {
 	Notify Notifier
 	// CallerOf is who called, for the arguments one call was handed (stdio.CallerOf); nil proves nobody.
 	CallerOf func(map[string]any) string
+	// Log is the tracker's record, the log `changes` (novox/hq ADR 0300); the bucket `issues` is its fold.
+	Log      Log
 	groom    grooming
+	held     *memoryView
+	heldOnce sync.Once
 }
 
 // The events this module says, one per kind of change.
@@ -122,9 +127,10 @@ type Opened struct {
 	Warning   string   `json:"warning,omitempty"`
 }
 
-// Open checks an issue, then takes the next number and writes the issue under it (to-be 53 §4). A refused
-// issue takes no number. A number taken and then not written — the bus failing between the two — stays
-// unused, and the answer says which: numbers may skip, never repeat.
+// Open checks an issue, then takes the next number, appends the issue's first operation to the log at expect
+// 0, and writes its fold under its number (to-be 53 §4, novox/hq ADR 0300 §2). An issue refused at validation
+// takes no number. A number taken and then not taken by the log — the bus failing between the two, or a full
+// log — is spent, and the open answers the failure: numbers may skip, never repeat.
 func (t *Tracker) Open(a OpenAsk) (*Opened, error) { return t.open(t.by(), a) }
 
 // open is Open as the one who acts.
@@ -174,9 +180,33 @@ func (t *Tracker) open(by string, a OpenAsk) (*Opened, error) {
 	if record != "" {
 		is.Record = RecordOf(n, record)
 	}
-	if err := t.create(&is); err != nil {
-		return nil, err
+	// The first operation, at expect 0 (ADR 0300 §2): one that fails after the number was taken spends the
+	// number, and the open answers the failure.
+	var seq uint64
+	if there, _, rerr := read(t.Store, n); rerr == nil && there.Opening != is.Opening {
+		// Kept from before the switch and not carried yet: the log does not know it, the bucket does.
+		err = errBegun
+	} else {
+		seq, err = t.begin(&is, Act{Op: OpOpened, By: by})
 	}
+	if errors.Is(err, errBegun) {
+		t.Logf("[mesh-issues] the counter handed out %d and an issue %d is already kept: not overwritten", n, n)
+		return nil, fmt.Errorf("a fault, not your mistake: the counter handed out %d, and an issue %d is already kept. "+
+			"Nothing was overwritten and nothing was opened; %d stays unused. Open it again for the next number, "+
+			"and tell the operator the counter is behind the issues kept", n, n, n)
+	}
+	if errors.Is(err, errUnsure) {
+		t.Logf("[mesh-issues] number %d was taken and the log's answer to the issue was lost: %v", n, err)
+		return nil, fmt.Errorf("number %d was taken and the log did not confirm the issue (%v): it may have been written. "+
+			"Call get %d before opening it again; if there is no issue %d, it is spent", n, err, n, n)
+	}
+	if err != nil {
+		t.Logf("[mesh-issues] number %d was taken and the log did not take the issue: %v", n, err)
+		return nil, fmt.Errorf("number %d was taken and the log did not take the issue (%v): nothing was opened and %d "+
+			"is spent. Open it again for the next number", n, err, n)
+	}
+	is.Folded = seq
+	t.store(n, &is, 0)
 	out := &Opened{Number: n, Title: title, Status: is.Status, LocatedIn: located, Record: is.Record, By: is.OpenedBy}
 	out.Said = fmt.Sprintf("Opened issue %d: %s.", n, title)
 	if is.Record != "" {
@@ -188,45 +218,6 @@ func (t *Tracker) open(by string, a OpenAsk) (*Opened, error) {
 	return out, nil
 }
 
-// create writes a new issue under its number, never over one: a key that is there already is a fault said
-// loudly, and the number stays unused.
-func (t *Tracker) create(is *Issue) error {
-	key := keyOf(is.Number)
-	var err error
-	for i := 0; i < createAttempts; i++ {
-		if i > 0 {
-			t.Sleep(jitter(time.Duration(i) * 50 * time.Millisecond))
-		}
-		_, err = t.Store.Create(IssuesState, key, is)
-		if err == nil {
-			return nil
-		}
-		if errors.Is(err, stdio.ErrChanged) {
-			// Ours, written by an attempt whose answer was lost?
-			there, _, rerr := read(t.Store, is.Number)
-			if rerr != nil {
-				t.Logf("[mesh-issues] issue %d is kept and could not be read back: %v", is.Number, rerr)
-				return fmt.Errorf("issue %d may have been written by this open: the key is taken and could not be read "+
-					"back (%v). Call get %d before opening it again, so it is not opened twice", is.Number, rerr, is.Number)
-			}
-			if there.Opening == is.Opening {
-				return nil
-			}
-			t.Logf("[mesh-issues] the counter handed out %d and an issue %d is already kept: not overwritten", is.Number, is.Number)
-			return fmt.Errorf("a fault, not your mistake: the counter handed out %d, and an issue %d is already kept. "+
-				"Nothing was overwritten and nothing was opened; %d stays unused. Open it again for the next number, "+
-				"and tell the operator the counter is behind the issues kept", is.Number, is.Number, is.Number)
-		}
-	}
-	// A write that timed out may still have landed: say so, never "nothing was opened".
-	t.Logf("[mesh-issues] number %d was taken and the issue's write was not confirmed: %v", is.Number, err)
-	if there, _, rerr := read(t.Store, is.Number); rerr == nil && there.Opening == is.Opening {
-		return nil
-	}
-	return fmt.Errorf("number %d was taken and the issue's write was not confirmed (%v): it may have been written. "+
-		"Call get %d before opening it again; if there is no issue %d, it stays unused", is.Number, err, is.Number, is.Number)
-}
-
 // next takes the next issue number: the counter advanced only at the revision read, so two openers — in
 // one instance or two — never take one number (ADR 0275 §3).
 func (t *Tracker) next() (uint64, error) {
@@ -285,38 +276,6 @@ func (t *Tracker) seed() error {
 	return nil
 }
 
-// change reads an issue, applies f, and writes it only at the revision read; a write another writer won
-// is read and applied again, so no note, link or move is lost and none is applied to a stale issue.
-func (t *Tracker) change(n uint64, f func(*Issue) error) (*Issue, error) {
-	for i := 0; i < issueAttempts; i++ {
-		is, rev, err := read(t.Store, n)
-		if err != nil {
-			return nil, err
-		}
-		if err := f(is); err != nil {
-			return nil, err
-		}
-		raw, err := json.Marshal(is)
-		if err != nil {
-			return nil, err
-		}
-		if len(raw) > MostIssueBytes {
-			return nil, fmt.Errorf("issue %d would be %d bytes; one issue is kept in at most %d (256 KiB). Nothing "+
-				"was changed: say it shorter, or in a document the issue links", n, len(raw), MostIssueBytes)
-		}
-		_, err = t.Store.Update(IssuesState, keyOf(n), json.RawMessage(raw), rev)
-		if err == nil {
-			return is, nil
-		}
-		if !errors.Is(err, stdio.ErrChanged) {
-			return nil, fmt.Errorf("issue %d could not be written (%v); nothing was changed", n, err)
-		}
-		t.Sleep(jitter(time.Duration(i+1) * 5 * time.Millisecond))
-	}
-	return nil, fmt.Errorf("issue %d was changed by another caller %d times while this change was made; nothing was "+
-		"changed. Try again", n, issueAttempts)
-}
-
 // Get is one issue whole.
 func (t *Tracker) Get(n uint64) (*Issue, error) {
 	is, _, err := read(t.Store, n)
@@ -451,7 +410,7 @@ func (t *Tracker) move(by string, a MoveAsk) (*Moved, error) {
 	var from string
 	var changed []string
 	var work workOutcome
-	is, err := t.change(a.Number, func(is *Issue) error {
+	is, err := t.change(a.Number, Act{Op: EventMoved, By: by}, func(is *Issue) error {
 		changed = nil
 		from = is.Status
 		at := t.now()
@@ -610,7 +569,7 @@ func (t *Tracker) note(by string, n uint64, text string) (*Noted, error) {
 		return nil, err
 	}
 	note := Note{By: by, At: t.now(), Text: text}
-	is, err := t.change(n, func(is *Issue) error {
+	is, err := t.change(n, Act{Op: EventNoted, By: by}, func(is *Issue) error {
 		is.Notes = append(is.Notes, note)
 		touch(is, by, note.At)
 		return nil
@@ -680,7 +639,7 @@ func (t *Tracker) link(by string, a LinkAsk) (*Linked, error) {
 		}
 	}
 	at := t.now()
-	is, err := t.change(a.Number, func(is *Issue) error {
+	is, err := t.change(a.Number, Act{Op: EventLinked, By: by}, func(is *Issue) error {
 		for i, l := range is.Links {
 			if l.Kind == kind && l.Target == target {
 				if !a.Remove {
@@ -727,6 +686,10 @@ type ListAsk struct {
 	Ready *bool
 	// Flag keeps only the issues delivered and not resolved, or the abandoned ones (novox/hq issue 483).
 	Flag string
+	// Search ranks the issues whose title, report or notes hold its words, best first (novox/hq ADR 0300 §5).
+	Search string
+	// Before keeps only the issues numbered below it: the cursor that pages the newest-first list.
+	Before uint64
 }
 
 // Row is one issue in a list: short enough to read in a chat.
@@ -755,12 +718,15 @@ type Row struct {
 
 // Listed is the answer to a list.
 type Listed struct {
-	Rows    []Row  `json:"rows"`
-	Matched int    `json:"matched"`
-	Said    string `json:"said"`
+	Rows    []Row `json:"rows"`
+	Matched int   `json:"matched"`
+	// Next is the before that continues the list where this page ends; 0 when nothing more matches.
+	Next uint64 `json:"next,omitempty"`
+	Said string `json:"said"`
 }
 
-// List is the issues that match, newest first.
+// List is the issues that match, newest first, or ranked by a search: from memory once it is loaded on start
+// (novox/hq ADR 0300 §5).
 func (t *Tracker) List(a ListAsk) (*Listed, error) {
 	status := strings.TrimSpace(a.Status)
 	if status != "" && !known(status) {
@@ -795,6 +761,19 @@ func (t *Tracker) List(a ListAsk) (*Listed, error) {
 			return nil, err
 		}
 	}
+	search := ""
+	if strings.TrimSpace(a.Search) != "" {
+		if search, err = singleLine("search", a.Search, MostPhrase); err != nil {
+			return nil, err
+		}
+		if len(tokens(search)) == 0 {
+			return nil, fmt.Errorf("search %q holds no word of two letters or digits", search)
+		}
+		if a.Before != 0 {
+			return nil, errors.New("before pages the newest-first list; a search is ranked best first, so give limit, " +
+				"or more words, instead")
+		}
+	}
 	limit := a.Limit
 	if limit <= 0 {
 		limit = 50
@@ -806,10 +785,30 @@ func (t *Tracker) List(a ListAsk) (*Listed, error) {
 	}
 	now, staleAfter := t.now(), t.workerStaleAfter()
 	var numbers []uint64
-	for n := range all {
-		numbers = append(numbers, n)
+	if search != "" {
+		var hits []hit
+		if v := t.view(); v.ready() {
+			hits = v.search(search)
+		} else {
+			x := newIndex()
+			for _, is := range all {
+				x.add(is)
+			}
+			hits = x.search(search)
+		}
+		for _, h := range hits {
+			if all[h.Number] != nil {
+				numbers = append(numbers, h.Number)
+			}
+		}
+	} else {
+		for n := range all {
+			if a.Before == 0 || n < a.Before {
+				numbers = append(numbers, n)
+			}
+		}
+		sort.Slice(numbers, func(i, j int) bool { return numbers[i] > numbers[j] })
 	}
-	sort.Slice(numbers, func(i, j int) bool { return numbers[i] > numbers[j] })
 	out := &Listed{Rows: []Row{}}
 	lower := strings.ToLower(phrase)
 	for _, n := range numbers {
@@ -849,13 +848,23 @@ func (t *Tracker) List(a ListAsk) (*Listed, error) {
 			out.Rows = append(out.Rows, row)
 		}
 	}
+	below := ""
+	if a.Before != 0 {
+		below = fmt.Sprintf(" below %d", a.Before)
+	}
 	switch {
 	case out.Matched == 0:
-		out.Said = "No issue matches."
+		out.Said = "No issue matches" + below + "."
+	case search != "" && out.Matched > len(out.Rows):
+		out.Said = fmt.Sprintf("%d issues match the search; the best %d are shown, best first.", out.Matched, len(out.Rows))
+	case search != "":
+		out.Said = fmt.Sprintf("%d issues match the search, best first.", out.Matched)
 	case out.Matched > len(out.Rows):
-		out.Said = fmt.Sprintf("%d issues match; the newest %d are shown.", out.Matched, len(out.Rows))
+		out.Next = out.Rows[len(out.Rows)-1].Number
+		out.Said = fmt.Sprintf("%d issues match%s; the newest %d are shown. The next page is before %d.", out.Matched,
+			below, len(out.Rows), out.Next)
 	default:
-		out.Said = fmt.Sprintf("%d issues match, newest first.", out.Matched)
+		out.Said = fmt.Sprintf("%d issues match%s, newest first.", out.Matched, below)
 	}
 	return out, nil
 }