Merge pull request 'Progress reads only a design's head, so a long design never takes the progress view down (issue 1947)' (#623) from fix/1947-progress-bounded-read into main

Why: with no argument the verb answered every ask of the month, 737,934 characters, more than a session reads.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i

modules/mesh-knowledge/cmd/mesh-knowledge/bus_test.go

@@ -107,9 +107,12 @@ func newBusWorld(t *testing.T) *busWorld {
 		return map[string]any{"folders": f, "documents": d}, err
 	})
 	answer("records.records_read", func(body json.RawMessage) (any, error) {
-		var a struct{ Path string }
+		var a struct {
+			Path string
+			Head int
+		}
 		_ = json.Unmarshal(body, &a)
-		c, commit, err := records.Read(a.Path)
+		c, commit, err := records.ReadHead(a.Path, a.Head)
 		return map[string]any{"content": c, "commit": commit}, err
 	})
 	j := &judge{}

modules/mesh-knowledge/cmd/mesh-knowledge/design.go

@@ -24,8 +24,26 @@ type Design struct {
 	Status     string
 	Decisions  []string
 	Milestones []Milestone
+	// Unread says why the design's frontmatter and title could not be read, when they could not (novox/hq issue
+	// 1947); the progress answer carries it as a warning beside a tree built from the tracker's milestone records.
+	Unread string
 }
 
+// unreadWarnings says each design whose head was not read, so none drops out of an answer silently.
+func unreadWarnings(ds []*Design) []string {
+	var out []string
+	for _, d := range ds {
+		if d.Unread != "" {
+			out = append(out, d.Unread)
+		}
+	}
+	return out
+}
+
+// DesignHeadBytes is how much of a design is read: its frontmatter and its title lead it, and its milestones are the
+// tracker's records, so the rest is never needed.
+const DesignHeadBytes = 16 << 10
+
 // Milestone is one milestone of a design, as the issue tracker keeps it.
 type Milestone struct {
 	ID      string
@@ -51,12 +69,13 @@ func parseDesign(path, content string) *Design {
 		d.Number, _ = strconv.Atoi(m[1])
 	}
 	lines := strings.Split(content, "\n")
-	body := 0
+	body, closed := 0, false
 	if len(lines) > 0 && strings.TrimSpace(lines[0]) == "---" {
 		for i := 1; i < len(lines); i++ {
 			l := lines[i]
 			if strings.TrimSpace(l) == "---" {
 				body = i + 1
+				closed = true
 				break
 			}
 			if v, ok := strings.CutPrefix(l, "status:"); ok {
@@ -67,6 +86,9 @@ func parseDesign(path, content string) *Design {
 			}
 		}
 	}
+	if len(lines) > 0 && strings.TrimSpace(lines[0]) == "---" && !closed {
+		d.Unread = fmt.Sprintf("the frontmatter of %s does not close in the part read: its state and decisions are not known here", path)
+	}
 	for i := body; i < len(lines) && d.Title == ""; i++ {
 		if m := titleLine.FindStringSubmatch(strings.TrimSpace(lines[i])); m != nil {
 			d.Title = strings.TrimSpace(m[1])
@@ -151,12 +173,22 @@ func (k *KB) designs(all bool, number int) ([]*Design, string, error) {
 		}
 		d := c.byPath[p]
 		if d == nil {
-			content, _, err := k.Records.Read(p)
+			var content string
+			var err error
+			if h, ok := k.Records.(HeadReader); ok {
+				content, _, err = h.ReadHead(p, DesignHeadBytes)
+			} else {
+				content, _, err = k.Records.Read(p)
+			}
 			if err != nil {
-				return nil, "", fmt.Errorf("the records module does not read %s: %v", p, err)
+				d = parseDesign(p, "")
+				d.Unread = fmt.Sprintf("the records module does not read %s (%v): its title and state are not known here", p, err)
+			} else {
+				d = parseDesign(p, content)
+			}
+			if d.Unread == "" {
+				c.byPath[p] = d
 			}
-			d = parseDesign(p, content)
-			c.byPath[p] = d
 			k.progReads++
 		}
 		out = append(out, d)

modules/mesh-knowledge/cmd/mesh-knowledge/hq.go

@@ -22,6 +22,12 @@ type Records interface {
 	Read(path string) (content, commit string, err error)
 }
 
+// HeadReader is Records that can read only the first part of a document: a caller that needs a frontmatter or a
+// title reads a long document through it, bounded, never whole (novox/hq issue 1947).
+type HeadReader interface {
+	ReadHead(path string, bytes int) (content, commit string, err error)
+}
+
 // busRecords is Records over the bus: the records module's tools, called as this module.
 type busRecords struct{}
 
@@ -58,8 +64,18 @@ func (busRecords) List(folder string) ([]string, []string, error) {
 	return f.Folders, f.Documents, nil
 }
 
-func (busRecords) Read(path string) (string, string, error) {
+func (busRecords) Read(path string) (string, string, error) { return busRead(path, 0) }
-	raw, err := stdio.Ask("records.records_read", map[string]any{"path": path})
+
+func (busRecords) ReadHead(path string, bytes int) (string, string, error) {
+	return busRead(path, bytes)
+}
+
+func busRead(path string, head int) (string, string, error) {
+	args := map[string]any{"path": path}
+	if head > 0 {
+		args["head"] = head
+	}
+	raw, err := stdio.Ask("records.records_read", args)
 	if err != nil {
 		return "", "", err
 	}
@@ -118,11 +134,19 @@ func (d dirRecords) List(folder string) ([]string, []string, error) {
 	return folders, docs, nil
 }
 
-func (d dirRecords) Read(path string) (string, string, error) {
+func (d dirRecords) Read(path string) (string, string, error) { return d.ReadHead(path, 0) }
+
+func (d dirRecords) ReadHead(path string, head int) (string, string, error) {
 	raw, err := os.ReadFile(filepath.Join(d.dir, filepath.FromSlash(path)))
 	if err != nil {
 		return "", "", err
 	}
+	if head > 0 && head < len(raw) {
+		raw = raw[:head]
+		if i := strings.LastIndexByte(string(raw), '\n'); i > 0 {
+			raw = raw[:i+1]
+		}
+	}
 	commit, _ := d.Standing()
 	return string(raw), commit, nil
 }

modules/mesh-knowledge/cmd/mesh-knowledge/progress.go

@@ -329,6 +329,12 @@ func (k *KB) progressDesign(p *Progress, number int, milestone string) error {
 		return err
 	}
 	d := ds[0]
+	if d.Unread != "" {
+		p.Warnings = append(p.Warnings, d.Unread)
+		if d.Title == "" {
+			d.Title = fmt.Sprintf("design %d", d.Number)
+		}
+	}
 	if len(d.Milestones) == 0 {
 		return fmt.Errorf("design %d has no milestone records in the issue tracker: seed them from its table with "+
 			"issue-tracker.import milestones %d", number, number)
@@ -387,6 +393,7 @@ func (k *KB) progressMesh(p *Progress) error {
 		return err
 	}
 	p.Source = fmt.Sprintf("the issue tracker's milestone records, and the designs in %s at hq %s", DesignsFolder, short(commit))
+	p.Warnings = append(p.Warnings, unreadWarnings(ds)...)
 	var with []*Design
 	var without []string
 	for _, d := range ds {
@@ -514,6 +521,7 @@ func (k *KB) progressRecord(p *Progress, kind string, n int) error {
 		return err
 	}
 	p.Source = fmt.Sprintf("the knowledge base, the designs at hq %s and the issue tracker's links", short(commit))
+	p.Warnings = append(p.Warnings, unreadWarnings(ds)...)
 	var designs []string
 	for _, d := range ds {
 		for _, dn := range d.Decisions {

modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go

@@ -42,6 +42,15 @@ func (w withDesigns) Read(path string) (string, string, error) {
 	return c, commit, err
 }
 
+func (w withDesigns) ReadHead(path string, n int) (string, string, error) {
+	if c, commit, err := w.base.ReadHead(path, n); err == nil {
+		return c, commit, nil
+	}
+	c, _, err := w.designs.ReadHead(path, n)
+	commit, _ := w.base.Standing()
+	return c, commit, err
+}
+
 // seatsWorld plays the issue tracker and mesh-delivery from the captured answers, and counts what it is asked.
 type seatsWorld struct {
 	mu     sync.Mutex
@@ -142,6 +151,19 @@ func (c *countingRecords) Read(p string) (string, string, error) {
 	return content, c.commit, err
 }
 
+func (c *countingRecords) ReadHead(p string, n int) (string, string, error) {
+	c.reads++
+	content, _, err := c.dirRecords.ReadHead(p, n)
+	return content, c.commit, err
+}
+
+// brokenRead is Records whose reads fail as the records module's do for a document longer than it answers whole.
+type brokenRead struct{ dirRecords }
+
+func (brokenRead) ReadHead(p string, n int) (string, string, error) {
+	return "", "", fmt.Errorf("%s is longer than the records module answers whole", p)
+}
+
 // 1947/AC2, as issue 1781 changed it: a design's title, state and decisions are hq's; its milestones are the issue
 // tracker's records, ordered as its table, an id found with spaces or hyphens, and a closed one holding no issue done.
 func TestADesignsMilestonesAreTheTrackersRecords(t *testing.T) {
@@ -316,6 +338,18 @@ func TestProgressOfADesignMinimalAndDetailed(t *testing.T) {
 	if big, _ := k.Progress(ProgressAsk{Of: "design 47", Bound: 8192}); big.Cut != "" || big.Tree != p.Tree {
 		t.Errorf("a minimal answer at bound 8192 is cut: %q", big.Cut)
 	}
+	// A design whose head the records module cannot read is said in a warning beside the tracker-built tree, never a failure.
+	kb := &KB{Records: brokenRead{progressRecords()}, Seats: newSeatsWorld(t).ask, Now: k.Now}
+	if bp, err := kb.Progress(ProgressAsk{Of: "design 47"}); err != nil || !strings.Contains(strings.Join(bp.Warnings, "\n"), "does not read") ||
+		!strings.Contains(bp.Tree, "design 47") || !strings.Contains(bp.Tree, "R6") {
+		t.Fatalf("an unreadable design: %v %+v", err, bp)
+	}
+	if mp, err := kb.Progress(ProgressAsk{Of: "mesh"}); err != nil || !strings.Contains(strings.Join(mp.Warnings, "\n"), "does not read") {
+		t.Fatalf("the mesh level drops an unread design silently: %v %v", err, mp.Warnings)
+	}
+	if d := parseDesign(DesignsFolder+"/47-x.md", "---\nstatus: in-progress\n"); d.Unread == "" {
+		t.Errorf("a frontmatter cut before its close is not said")
+	}
 	lines := strings.Split(p.Tree, "\n")
 	if !strings.HasPrefix(lines[0], "✗ design 47 · Delivery: from a commit to delivered — ") {
 		t.Fatalf("first line %q", lines[0])

modules/messenger/cmd/messenger/asks.go

@@ -1418,27 +1418,35 @@ func (h *Holder) AsksNow() []AskView {
 }
 
 // AsksQuery narrows what the tool `operator-channel.asks` answers (issue 595): Key one ask, State open (open and held) or
-// ended, Since the asks opened at or after that time, Bound the most bytes of the answer's asks. The zero query keeps the
+// ended (all takes both), Since the asks opened at or after that time, Bound the most bytes of the answer's asks. With no
-// whole answer, as before.
+// argument at all the query is the open asks within DefaultAsksBound (issue 1974), and the answer says so.
 type AsksQuery struct {
 	Key, State string
 	Since      time.Time
 	Bound      int
+	// Defaulted is set by AsksQueryOf when no key, state or since was given: the open asks only.
+	Defaulted bool
 }
 
+// DefaultAsksBound is the most bytes of asks an answer holds when the caller gives no bound (issue 1974: the whole of a
+// month's asks was 737,934 characters, more than a session reads).
+const DefaultAsksBound = 32768
+
 // AsksAnswer is the answer of a narrowed query: the asks kept, how many the filters matched, and, when the bound cut the
 // answer short, that it did and how.
 type AsksAnswer struct {
 	Asks    []AskView `json:"asks"`
 	Matched int       `json:"matched"`
 	Cut     string    `json:"cut,omitempty"`
+	// Default says that no key, state or since was given, so only the open asks are answered, and how to ask for more.
+	Default string `json:"default,omitempty"`
 }
 
 // AsksQueryOf reads the arguments of the tool; an empty set is the zero query.
 func AsksQueryOf(a map[string]any) (AsksQuery, error) {
 	q := AsksQuery{Key: strArg(a, "key"), State: strArg(a, "state")}
-	if q.State != "" && q.State != "open" && q.State != "ended" {
+	if q.State != "" && q.State != "open" && q.State != "ended" && q.State != "all" {
-		return q, errorf("state is open or ended, not %q", q.State)
+		return q, errorf("state is open, ended or all, not %q", q.State)
 	}
 	if since := strArg(a, "since"); since != "" {
 		t, err := time.Parse(time.RFC3339, since)
@@ -1453,6 +1461,12 @@ func AsksQueryOf(a map[string]any) (AsksQuery, error) {
 		}
 		q.Bound = int(v)
 	}
+	if q.Key == "" && q.State == "" && q.Since.IsZero() {
+		q.State, q.Defaulted = "open", true
+	}
+	if q.Bound == 0 {
+		q.Bound = DefaultAsksBound
+	}
 	return q, nil
 }
 
@@ -1461,6 +1475,10 @@ func AsksQueryOf(a map[string]any) (AsksQuery, error) {
 func (q AsksQuery) Narrow(all []AskView) AsksAnswer {
 	var out AsksAnswer
 	out.Asks = []AskView{}
+	if q.Defaulted {
+		out.Default = "no key, state or since was given, so only the open asks are shown: state ended or all for the ended ones, " +
+			"key for one ask, since for the asks opened from a time"
+	}
 	var kept []AskView
 	for _, v := range all {
 		open := v.State == StateOpen || v.State == StateHeld

modules/messenger/cmd/messenger/held_test.go

@@ -143,7 +143,7 @@ func TestAHeldAskNamesEveryKindAndWhyWhenNoChannelIsReady(t *testing.T) {
 }
 
 // Issue 595: operator-channel.asks answers one ask, the open or the ended, those opened from a time, and no more bytes than a
-// bound, said when it cut the answer short; with no argument it keeps the whole answer.
+// bound, said when it cut the answer short; with no argument it answers the open asks only, within a default bound, and says so (issue 1974).
 func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) {
 	view := func(key, state, opened string) AskView {
 		return AskView{Key: key, Asker: "mesh-delivery", Headline: "Approve " + key, State: state, Opened: opened}
@@ -159,13 +159,15 @@ func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) {
 		cut     bool
 		err     bool
 	}{
-		{"no argument is the zero query", map[string]any{}, nil, 0, false, false},
+		{"no argument is the open asks, said", map[string]any{}, []string{"a.3", "a.2"}, 2, false, false},
+		{"state all is every ask", map[string]any{"state": "all"}, []string{"a.3", "a.2", "a.1", "a.0"}, 4, false, false},
+		{"a key finds an ended ask whatever its state", map[string]any{"key": "a.0"}, []string{"a.0"}, 1, false, false},
 		{"key", map[string]any{"key": "a.1"}, []string{"a.1"}, 1, false, false},
 		{"state open takes open and held", map[string]any{"state": "open"}, []string{"a.3", "a.2"}, 2, false, false},
 		{"state ended", map[string]any{"state": "ended"}, []string{"a.1", "a.0"}, 2, false, false},
 		{"since", map[string]any{"since": "2026-11-02T12:00:00Z"}, []string{"a.3", "a.2"}, 2, false, false},
 		{"bound cuts and says so", map[string]any{"bound": float64(2*len(one) + 10), "state": "open"}, []string{"a.3", "a.2"}, 2, false, false},
-		{"a bound under the asks cuts", map[string]any{"bound": float64(len(one) + 5)}, []string{"a.3"}, 4, true, false},
+		{"a bound under the asks cuts", map[string]any{"bound": float64(len(one) + 5)}, []string{"a.3"}, 2, true, false},
 		{"a state that is neither", map[string]any{"state": "done"}, nil, 0, false, true},
 		{"a time that is not one", map[string]any{"since": "yesterday"}, nil, 0, false, true},
 	} {
@@ -177,13 +179,10 @@ func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) {
 		if tc.err {
 			continue
 		}
-		if tc.want == nil {
-			if q != (AsksQuery{}) {
-				t.Errorf("%s: %+v is not the zero query, which keeps today's answer", tc.name, q)
-			}
-			continue
-		}
 		got := q.Narrow(all)
+		if (got.Default != "") != (tc.args["key"] == nil && tc.args["state"] == nil && tc.args["since"] == nil) || q.Bound == 0 {
+			t.Errorf("%s: default %q, bound %d: only no argument is defaulted, and every query is bounded", tc.name, got.Default, q.Bound)
+		}
 		var keys []string
 		for _, v := range got.Asks {
 			keys = append(keys, v.Key)

modules/messenger/cmd/messenger/main.go

@@ -734,12 +734,13 @@ func tools(h *Holder, l *listening) []stdio.Tool {
 		{Name: "operator-channel.asks",
 			Description: "The asks this router holds (novox/hq ADR 0259): open (or held, with since when and why no channel can carry it yet) first, then the last month's — each " +
 				"one's asker, headline, state, when it opened and expires, the channels it was offered on, how it " +
-				"ended (the warrant: who chose what, through which channel), and whether its asker was told. With no argument every " +
+				"ended (the warrant: who chose what, through which channel), and whether its asker was told. With no argument " +
-				"ask, which is large; filter it: key for one ask, state open or ended, since (a time, RFC 3339) for the asks opened " +
+				"only the open asks, within 32768 bytes; the answer says what it cut and how to ask for more: key for one ask " +
-				"from then, and bound, the most bytes of the answer, which says when it cut it short.",
+				"(any state), state open, ended or all, since (a time, RFC 3339) for the asks opened from then, and bound, the " +
+				"most bytes of the answer.",
 			Input: map[string]any{
-				"key":   str("one ask, by its key (asker.id); none for every ask"),
+				"key":   str("one ask, by its key (asker.id); none for the open asks"),
-				"state": str("open (open and held) or ended; none for both"),
+				"state": str("open (open and held), ended or all; none is open when no key or since is given, else both"),
 				"since": str("only the asks opened at or after this time, RFC 3339"),
 				"bound": map[string]any{"type": "integer", "description": "the most bytes of the asks in the answer; said when it cuts the answer short"}},
 			Run: func(a map[string]any) (any, error) {
@@ -747,9 +748,6 @@ func tools(h *Holder, l *listening) []stdio.Tool {
 				if err != nil {
 					return nil, err
 				}
-				if q == (AsksQuery{}) {
-					return h.AsksNow(), nil
-				}
 				return q.Narrow(h.AsksNow()), nil
 			}},
 		{Name: "messenger_identities",

modules/records/cmd/records/main.go

@@ -52,9 +52,16 @@ func tools(r *Records) []stdio.Tool {
 				return r.Search(strArg(a, "query"), limit)
 			}},
 		{Name: "records_read",
-			Description: "One document, whole, by its path in the repository — a decision record, a design document, an issue report.",
+			Description: "One document by its path (whole, or its first part with head) in the repository — a decision record, a design document, an issue report.",
-			Input:       map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md")},
+			Input: map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md"),
-			Run:         func(a map[string]any) (any, error) { return r.Read(strArg(a, "path")) }},
+				"head": map[string]any{"type": "number", "description": "only the first this many bytes, cut at a line end (optional); the answer says partial"}},
+			Run: func(a map[string]any) (any, error) {
+				head := 0
+				if v, ok := a["head"].(float64); ok {
+					head = int(v)
+				}
+				return r.ReadHead(strArg(a, "path"), head)
+			}},
 		{Name: "records_list",
 			Description: "What a folder of the repository holds: its sub-folders and its documents. The root when no folder is named.",
 			Input:       map[string]any{"folder": str("a folder inside the repository (optional)")},

modules/records/cmd/records/records.go

@@ -322,12 +322,18 @@ type Document struct {
 	Path      string `json:"path"`
 	Content   string `json:"content"`
 	Truncated bool   `json:"truncated"`
-	Commit    string `json:"commit"`
+	// Partial is true when only the first part the caller asked for (head) is answered; Truncated stays the answer's own cap.
+	Partial bool   `json:"partial,omitempty"`
+	Commit  string `json:"commit"`
 }
 
 // Read is one document, whole, or its first part with a note when it is very long. The path stays inside
 // the checkout: `..` and absolute paths are refused, not resolved.
-func (r *Records) Read(p string) (Document, error) {
+func (r *Records) Read(p string) (Document, error) { return r.ReadHead(p, 0) }
+
+// ReadHead is Read, or with head above zero only the first head bytes of the document, cut at a line end, so a
+// caller that needs a frontmatter or a title reads a long document without its whole length (novox/hq issue 1947).
+func (r *Records) ReadHead(p string, head int) (Document, error) {
 	clean, ok := inside(p)
 	if !ok || clean == "" {
 		return Document{}, fmt.Errorf("%q is not a path inside the repository", p)
@@ -337,6 +343,15 @@ func (r *Records) Read(p string) (Document, error) {
 		return Document{}, fmt.Errorf("the repository holds no %s — `records_list` says what it holds", clean)
 	}
 	d := Document{Path: clean, Content: string(raw), Commit: r.Standing().Commit}
+	head = min(head, MostBytes)
+	if head > 0 && head < len(d.Content) {
+		cut := d.Content[:head]
+		if i := strings.LastIndex(cut, "\n"); i > 0 {
+			cut = cut[:i+1]
+		}
+		d.Content, d.Partial = cut, true
+		return d, nil
+	}
 	if len(d.Content) > MostBytes {
 		d.Content = d.Content[:MostBytes] + "\n\n[… truncated; the document is longer than this answer carries]"
 		d.Truncated = true

modules/records/cmd/records/records_test.go

@@ -70,6 +70,10 @@ func TestAPhraseInOneDesignDocumentComesBackFromWhereItIsWritten(t *testing.T) {
 	if err != nil || !strings.Contains(doc.Content, "We decided the thing") || doc.Truncated {
 		t.Fatalf("%+v %v", doc, err)
 	}
+	if head, err := r.ReadHead("02-DECISIONS/0001-a-decision.md", 30); err != nil || !head.Partial || head.Truncated || len(head.Content) > 30 ||
+		!strings.HasSuffix(head.Content, "\n") || !strings.HasPrefix(doc.Content, head.Content) {
+		t.Fatalf("a head read: %+v %v", head, err)
+	}
 	root, _ := r.List("")
 	if strings.Join(root.Folders, ",") != "02-DECISIONS,03-DESIGN" || strings.Join(root.Documents, ",") != "README.md" {
 		t.Fatalf("%+v", root)