Merge pull request 'The knowledge base keeps the roadmap: a research effort can be an idea, and at most three are active (issue 1970, ADR 0629)' (#609) from fix/1970-research-ideas into main

modules/mesh-knowledge/cmd/mesh-knowledge/answers.go

@@ -97,7 +97,7 @@ func held(r *Record, c Clause) string {
 
 // Served says whether a clause may be served as holding: in force or changed, and not drafted.
 func Served(r *Record, c Clause) bool {
-	return r.Accepted() && c.Source != SourceDrafted && (c.Status == ClauseInForce || c.Status == ClauseChanged)
+	return r.Accepted() && r.Status != StatusIdea && c.Source != SourceDrafted && (c.Status == ClauseInForce || c.Status == ClauseChanged)
 }
 
 // AnswerClause is one clause in an answer.
@@ -219,6 +219,10 @@ func (k *KB) Ask(question string, bound int) (*Asked, error) {
 			continue
 		}
 		r := e.Current
+		if r.Status == StatusIdea {
+			// An idea is not what holds (ADR 0629 §2): ask answers nothing from it; list at idea shows the roadmap.
+			continue
+		}
 		ar := AnswerRecord{Record: g.key.String(), Title: r.Title, Status: r.Status, Read: readHow(g.key), Since: k.since(ctx, g.key)}
 		sort.Ints(g.clauses)
 		for _, n := range g.clauses {
@@ -304,6 +308,8 @@ type Got struct {
 	Reasoning  []Section      `json:"reasoning,omitempty"`
 	Insights   []Insight      `json:"insights,omitempty"`
 	Pending    string         `json:"pending,omitempty"`
+	// Idea is said of a research effort kept as an idea: it is not what holds (ADR 0629 §2).
+	Idea    string `json:"idea,omitempty"`
 	Waiting string `json:"waiting,omitempty"`
 	// Retracted are the retractions the operator approved; Retracting, one waiting for the operator.
 	Retracted   []RetractionRow `json:"retracted,omitempty"`
@@ -361,6 +367,9 @@ func (k *KB) Get(a GetAsk) (*Got, error) {
 	if r.Status == StatusReserved || r.Status == StatusWithdrawn {
 		g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
 	}
+	if r.Status == StatusIdea {
+		g.Idea = "an idea: research it again before anything rests on it"
+	}
 	if e.Pending != nil && a.Version == 0 {
 		g.Pending = fmt.Sprintf("version %d, a progressive insight or a new version of an active effort, waits for its review: history shows what it changes", e.PendingVersion)
 	}

modules/mesh-knowledge/cmd/mesh-knowledge/idea_test.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+// effort accepts a new research effort at the status, under a number reserved for it.
+func (w *world) effort(t *testing.T, status string) int {
+	t.Helper()
+	r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
+	if err != nil {
+		t.Fatal(err)
+	}
+	w.accepted(t, "author@g14", effortAsk(r.Number, status, Section{Name: "00-overview.md", Text: "What a roadmap entry would be."}))
+	return r.Number
+}
+
+// ADR 0629 §3, §4: at most three efforts are active; a fourth, new or taken up from an idea, is refused and the
+// refusal names the active ones; the limit is the setting's.
+func TestAtMostThreeResearchEffortsAreActive(t *testing.T) {
+	w := imported(t, newMemStore())
+	var active []int
+	for i := 0; i < 3; i++ {
+		active = append(active, w.effort(t, StatusActive))
+	}
+	idea := w.effort(t, StatusIdea)
+	if got := mustGet(t, w.k.Store, Key{KindResearch, idea}).Current.Status; got != StatusIdea {
+		t.Fatalf("an effort proposed as an idea is %s", got)
+	}
+	r, err := w.k.ReserveNumber("author@g14", KindResearch, "a fourth")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, ask := range map[string]ProposeAsk{
+		"a new effort":          effortAsk(r.Number, StatusActive, Section{Name: "00-overview.md", Text: "A fourth."}),
+		"an idea taken up":      effortAsk(idea, StatusActive, Section{Name: "00-overview.md", Text: "Taken up."}),
+		"a new effort, as idea": effortAsk(r.Number, StatusIdea, Section{Name: "00-overview.md", Text: "A fourth."}),
+	} {
+		_, err := w.k.Propose("author@g14", ask)
+		refused := err != nil && strings.Contains(err.Error(), "research-active-at-most")
+		if want := name != "a new effort, as idea"; refused != want {
+			t.Errorf("%s: refused %v, want %v (%v)", name, refused, want, err)
+		}
+		if refused {
+			for _, n := range active {
+				if !strings.Contains(err.Error(), Key{KindResearch, n}.String()) {
+					t.Errorf("%s: the refusal does not name research %d: %v", name, n, err)
+				}
+			}
+		}
+	}
+	// ADR 0629 §4: an active effort takes a new version at the limit (even over it); the setting raises the limit, and an idea taken up and
+	// still waiting for its review counts as active.
+	if _, err := w.k.Propose("author@g14", effortAsk(active[0], StatusActive, Section{Name: "00-overview.md", Text: "More."})); err != nil {
+		t.Fatalf("an active effort refused a new version at the limit: %v", err)
+	}
+	w.k.Settings = func() (Settings, error) { return Settings{ResearchActiveAtMost: 1}, nil }
+	if _, err := w.k.Propose("author@g14", effortAsk(active[1], StatusActive, Section{Name: "00-overview.md", Text: "Over the limit."})); err != nil {
+		t.Fatalf("an active effort refused a new version over the limit: %v", err)
+	}
+	w.k.Settings = func() (Settings, error) { return Settings{ResearchActiveAtMost: 4}, nil }
+	if _, err := w.k.Propose("author@g14", effortAsk(idea, StatusActive, Section{Name: "00-overview.md", Text: "Taken up."})); err != nil {
+		t.Fatalf("an idea taken up under a limit of 4: %v", err)
+	}
+	if _, err := w.k.Propose("author@g14", effortAsk(r.Number, StatusActive, Section{Name: "00-overview.md", Text: "A fifth."})); err == nil ||
+		!strings.Contains(err.Error(), "research-active-at-most") {
+		t.Fatalf("a fifth active (three accepted, one waiting) was not refused: %v", err)
+	}
+}
+
+// ADR 0629 §1, §2: list at idea is the roadmap; ask, holds and get never answer an idea as a record in force.
+func TestAnIdeaIsListedAndNeverServedAsTruth(t *testing.T) {
+	w := imported(t, newMemStore())
+	n := w.effort(t, StatusIdea)
+	got, err := w.k.List(ListAsk{Kind: KindResearch, Status: StatusIdea})
+	if err != nil || len(got.Rows) != 1 {
+		t.Fatalf("list at idea: %+v, %v", got, err)
+	}
+	asked, err := w.k.Ask("roadmap entry inference layer", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range asked.Records {
+		if r.Record == (Key{KindResearch, n}).String() {
+			t.Errorf("ask answered from an idea: %+v", r)
+		}
+	}
+	g, err := w.k.Get(GetAsk{Kind: KindResearch, Number: n})
+	if err != nil || !strings.Contains(g.Idea, "research it again") {
+		t.Fatalf("get does not say an idea: %+v, %v", g, err)
+	}
+	if Served(mustGet(t, w.k.Store, Key{KindResearch, n}).Current, Clause{Status: ClauseInForce}) {
+		t.Errorf("a clause of an idea is served")
+	}
+}

modules/mesh-knowledge/cmd/mesh-knowledge/kb.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -53,8 +54,14 @@ type Settings struct {
 	// ReservedFor is how long a reserved number waits for its proposal before it is withdrawn, as a duration
 	// such as 168h; DefaultReservedFor when empty.
 	ReservedFor string `json:"reserved-for,omitempty"`
+	// ResearchActiveAtMost is the most research efforts that may be active at once (ADR 0629); DefaultResearchActiveAtMost
+	// when absent or 0.
+	ResearchActiveAtMost int `json:"research-active-at-most,omitempty"`
 }
 
+// DefaultResearchActiveAtMost is how many research efforts may be active at once unless the settings say otherwise.
+const DefaultResearchActiveAtMost = 3
+
 // DefaultReservedFor is how long a reservation waits for its proposal unless the settings say otherwise.
 const DefaultReservedFor = 7 * 24 * time.Hour
 
@@ -248,7 +255,7 @@ type ProposeAsk struct {
 	NeedsOperator bool
 	// Brief is what the operator's ask shows (novox/hq issue 639).
 	Brief *Brief
-	// Status is a research effort's: active, graduated or abandoned.
+	// Status is a research effort's: active, graduated, abandoned or idea.
 	Status string
 	// Insight, for an accepted record, is what stood before: the proposal is a progressive insight.
 	Insight string
@@ -348,7 +355,7 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
 			}
 			// An active research effort is still being researched: it takes new versions freely, each reviewed as a
 			// proposal while the served version stays (novox/hq issue 844). Every other accepted record needs an insight.
-			revision := a.Kind == KindResearch && held.Current.Status == StatusActive && strings.TrimSpace(a.Insight) == ""
+			revision := a.Kind == KindResearch && (held.Current.Status == StatusActive || held.Current.Status == StatusIdea) && strings.TrimSpace(a.Insight) == ""
 			if strings.TrimSpace(a.Insight) == "" && !revision {
 				return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
 					"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
@@ -390,6 +397,11 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
 	if !current && a.Kind == KindResearch && r.Effort == "" {
 		r.Effort = held.Current.Status
 	}
+	if a.Kind == KindResearch && r.Effort == StatusActive && (held == nil || held.Current.Status != StatusActive) {
+		if why := k.activeLimitRefusal(ctx, key); why != "" {
+			refusals = append(refusals, Refusal{Rule: "research-active-at-most", Says: why})
+		}
+	}
 	// A new version of an active effort is new text: the dated rules hold it from the day it is proposed, while the
 	// record keeps its own date (novox/hq issue 844).
 	day := r.Date
@@ -434,6 +446,36 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
 	return &Proposed{Record: key.String(), Version: n, Status: StatusProposed, Says: says + "."}, nil
 }
 
+// activeLimitRefusal (ADR 0629 §4 names only a new effort or one taken up from an idea, never a new version of an active one; a check of the record set beside the one-record rules of ADR 0290 §4) says why a research effort may not be active now: the limit of efforts other than this one
+// already active (accepted at active, or proposed at active and waiting for review) is reached (ADR 0629). Empty
+// when it may.
+func (k *KB) activeLimitRefusal(ctx context.Context, key Key) string {
+	limit := k.settings().ResearchActiveAtMost
+	if limit <= 0 {
+		limit = DefaultResearchActiveAtMost
+	}
+	all, err := k.Store.All(ctx)
+	if err != nil {
+		return fmt.Sprintf("the active efforts cannot be counted (%v)", err)
+	}
+	var names []string
+	for _, e := range all {
+		if e.Current == nil || e.Current.Kind != KindResearch || e.Current.Key() == key {
+			continue
+		}
+		w := e.Working()
+		if e.Current.Status == StatusActive || (w != nil && w.Effort == StatusActive && (w.Status == StatusProposed || e.Pending != nil)) {
+			names = append(names, fmt.Sprintf("%s (%s)", e.Current.Key(), e.Current.Title))
+		}
+	}
+	if len(names) < limit {
+		return ""
+	}
+	sort.Strings(names)
+	return fmt.Sprintf("at most %d research efforts are active and %d already are: %s. Park one as an idea first, or propose this one as an idea",
+		limit, len(names), strings.Join(names, "; "))
+}
+
 // ---- acceptance ------------------------------------------------------------------------------------------
 
 // accept makes a reviewed version the one served: accepted, its clauses in force, every clause and record it

modules/mesh-knowledge/cmd/mesh-knowledge/list.go

@@ -28,7 +28,7 @@ const MostListResult = 500
 
 // AllStatuses are every status a record of either kind may hold.
 var AllStatuses = []string{StatusReserved, StatusProposed, StatusAccepted, StatusSuperseded, StatusWithdrawn,
-	StatusActive, StatusGraduated, StatusAbandoned, StatusRetracted}
+	StatusActive, StatusGraduated, StatusAbandoned, StatusIdea, StatusRetracted}
 
 // ListAsk narrows a list.
 type ListAsk struct {

modules/mesh-knowledge/cmd/mesh-knowledge/main.go

@@ -974,7 +974,7 @@ func Tools(k *KB, caller func(map[string]any) string) []stdio.Tool {
 				"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
 				"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
 				"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
-				"proposal is revised by proposing it again; an accepted record (an active research effort excepted, which takes new versions) changes only by a progressive insight (insight: what " +
+				"proposal is revised by proposing it again; an accepted record (an active research effort or an idea excepted, which take new versions; at most as many research efforts are active as the setting research-active-at-most allows, three by default) changes only by a progressive insight (insight: what " +
 				"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
 				"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
 				"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +

modules/mesh-knowledge/cmd/mesh-knowledge/record.go

@@ -37,13 +37,15 @@ const (
 	StatusActive     = "active"
 	StatusGraduated  = "graduated"
 	StatusAbandoned  = "abandoned"
+	// StatusIdea is a research effort kept on the roadmap: held and listed, never served as in force (ADR 0629 §1, §2).
+	StatusIdea = "idea"
 	// StatusRetracted is a record taken back whole on the operator's Approve (retract.go): it no longer holds, and
 	// its versions stay.
 	StatusRetracted = "retracted"
 )
 
 // ResearchStatuses are the statuses an accepted research effort may have.
-var ResearchStatuses = []string{StatusActive, StatusGraduated, StatusAbandoned}
+var ResearchStatuses = []string{StatusActive, StatusGraduated, StatusAbandoned, StatusIdea}
 
 // A clause's status (to-be 55 §3).
 const (
@@ -443,7 +445,7 @@ func (r *Record) Section(name string) string {
 // Accepted says the record holds: an accepted decision, or a research effort accepted at any of its statuses.
 func (r *Record) Accepted() bool {
 	switch r.Status {
-	case StatusAccepted, StatusActive, StatusGraduated, StatusAbandoned:
+	case StatusAccepted, StatusActive, StatusGraduated, StatusAbandoned, StatusIdea:
 		return true
 	}
 	return false

modules/mesh-knowledge/cmd/mesh-knowledge/rules.go

@@ -36,7 +36,7 @@ var PerRecordRules = []Rule{
 	{"topic", "a decision's topic is one of the reading order's", "records.py topics, index.py"},
 	{"date", "its date is a day, YYYY-MM-DD", "records.py insights, cycle.py deferrals"},
 	{"deciders", "a decision names who decided it", "records.py (deciders on every record)"},
-	{"research-status", "a research effort is active, graduated or abandoned", "cycle.py research"},
+	{"research-status", "a research effort is active, graduated, abandoned or idea", "cycle.py research"},
 	{"has-clause", "a decision has at least one clause", "records.py clauses"},
 	{"clause-numbering", "its clauses are numbered from 1, in order, each once", "records.py clauses"},
 	{"clause-lead", "every clause opens with a bold lead", "records.py clauses"},
@@ -160,7 +160,7 @@ func JudgeRecordOn(r *Record, held *Entry, lookup Lookup, ref *Reference, day st
 		refuse("deciders", "a decision names who decided it")
 	}
 	if !decision && !contains(ResearchStatuses, r.Effort) {
-		refuse("research-status", "%q is not active, graduated or abandoned", r.Effort)
+		refuse("research-status", "%q is not active, graduated, abandoned or idea", r.Effort)
 	}
 	if decision && len(r.Clauses) == 0 {
 		refuse("has-clause", "a decision is made of clauses, and this one has none")

modules/mesh-knowledge/files/settings.json

@@ -1,5 +1,6 @@
 {
   "judge-model": "sonnet",
   "drafting": "on",
-  "reserved-for": "168h"
+  "reserved-for": "168h",
+  "research-active-at-most": 3
 }

modules/mesh-knowledge/module.json

@@ -111,6 +111,12 @@
       "description": "How long a reserved number waits for its proposal before it is withdrawn, as a duration such as 168h; empty is the knowledge base's own default.",
       "pattern": "|[0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h)([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))*",
       "default": "168h"
+    },
+    "research-active-at-most": {
+      "type": "number",
+      "description": "The most research efforts that may be active at once; propose refuses a further one, which is parked as an idea instead. 0 or empty is 3.",
+      "minimum": 0,
+      "default": 3
     }
   },
   "resources": [
@@ -134,7 +140,7 @@
       "type": "file",
       "path": "${dir:state}/settings.json",
       "mode": "0600",
-      "takes-config": ["judge-model", "drafting", "reserved-for"],
+      "takes-config": ["judge-model", "drafting", "reserved-for", "research-active-at-most"],
       "merge": "json",
       "template": "files/settings.json"
     },