From ddb2ae942030bbb46e24ec7b83d3ed49438d5ab5 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 15:39:16 +0100 Subject: [PATCH 01/10] knowledge-base get shows the findings of the pending version's reviews (issue 1970) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-knowledge/README.md | 2 +- .../cmd/mesh-knowledge/active_effort_test.go | 29 ++++++++++++++++ .../cmd/mesh-knowledge/answers.go | 33 ++++++++++++++----- .../mesh-knowledge/cmd/mesh-knowledge/main.go | 2 +- .../cmd/mesh-knowledge/review_test.go | 9 +++++ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/modules/mesh-knowledge/README.md b/modules/mesh-knowledge/README.md index 5804bf92c..2c776eca5 100644 --- a/modules/mesh-knowledge/README.md +++ b/modules/mesh-knowledge/README.md @@ -16,7 +16,7 @@ operator, it is accepted only on the operator's Approve, given on a channel that | Verb | Takes | Answers | |---|---|---| | `ask` | `question` (at most 500 characters), `bound` | the records and clauses bearing on the question, ranked; each clause says whether it holds, what changed it, or that it is only drafted; each record says what changed it since and how to read its reasoning | -| `get` | `number`, `kind`, `reasoning`, `section`, `version`, `bound` | one record: clauses, links both ways, what changed it since, versions, reviews, the operator's ask; its reasoning only when asked | +| `get` | `number`, `kind`, `reasoning`, `section`, `version`, `bound` | one record: clauses, links both ways, what changed it since, versions, reviews, the pending version's reviews with their findings (`pending-reviews`), the operator's ask; its reasoning only when asked | | `list` | `kind`, `status`, `topic`, `flag` (`research-done` or `proposed-old`), `before` (a number, the cursor), `limit` | short rows, newest first, number by number: number, kind, title, status, topic, date, versions, and whether a proposal is pending; `next`, the `before` that reads on, when more match | | `findings` | `number` (optional) | the whole-set findings kept — each a rule's failure, the records it names, what it says, and when; with `number`, only the findings naming that record | | `holds` | `topic` or `number`, `bound` | the clauses in force for a topic of the reading order or for one record; never a drafted clause | diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/active_effort_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/active_effort_test.go index 3302dec07..3058781b8 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/active_effort_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/active_effort_test.go @@ -1,6 +1,8 @@ package main import ( + "context" + "fmt" "strings" "testing" "time" @@ -118,6 +120,33 @@ func TestAChangesVerdictLeavesTheServedVersionServedAndGraduatingNeedsABecameLin e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "" { t.Fatalf("after changes: pending %v, current %+v", e.Pending != nil, e.Current.Sections) } + // 1970/AC1: get with no version shows the pending version's findings, and says where the version stands. + g, err := w.k.Get(GetAsk{Kind: KindResearch, Number: r.Number}) + if err != nil { + t.Fatal(err) + } + if len(g.PendingReviews) != 1 || g.PendingReviews[0].Verdict != VerdictChanges || g.PendingReviews[0].Version == 0 || + !strings.Contains(g.PendingReviews[0].Findings, "say where the finding comes from") || len(g.Reviews) != 1 || + !strings.Contains(g.Pending, "changes asked") || !strings.Contains(g.Pending, "pending-reviews") { + t.Fatalf("pending findings not shown: %+v, pending %q, reviews %+v", g.PendingReviews, g.Pending, g.Reviews) + } + // 1970/AC3: under a small bound the last pending review is kept, and the cut is said. + e2 := mustGet(t, w.k.Store, key) + more := e2.Pending.Clone() + for i := 0; i < 3; i++ { + more.Reviews = append(more.Reviews, Review{By: "r", Verdict: VerdictChanges, Version: 9, Findings: strings.Repeat("a long finding ", 60) + fmt.Sprint(i)}) + } + if err := w.k.Store.Save(context.Background(), Version{By: "author@g14", Why: "test", Record: more}, false, false, e2.Revision); err != nil { + t.Fatal(err) + } + g, err = w.k.Get(GetAsk{Kind: KindResearch, Number: r.Number, Bound: LeastBound}) + if err != nil { + t.Fatal(err) + } + if len(g.PendingReviews) != 1 || !strings.HasSuffix(g.PendingReviews[0].Findings, "2") || + !strings.Contains(g.Cut, "earlier pending review") { + t.Fatalf("pending cut: %d kept, cut %q", len(g.PendingReviews), g.Cut) + } grad := effortAsk(r.Number, StatusGraduated, overview) if _, err := w.k.Propose("author@g14", grad); err == nil || !strings.Contains(err.Error(), "became") { t.Fatalf("graduated without a became link: %v", err) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/answers.go b/modules/mesh-knowledge/cmd/mesh-knowledge/answers.go index 7c42b1f43..4a9941fb3 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/answers.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/answers.go @@ -312,14 +312,17 @@ type Got struct { Idea string `json:"idea,omitempty"` Waiting string `json:"waiting,omitempty"` // Retracted are the retractions the operator approved; Retracting, one waiting for the operator. - Retracted []RetractionRow `json:"retracted,omitempty"` - Retracting string `json:"retracting,omitempty"` - Reviews []Review `json:"reviews,omitempty"` - Drafting *Drafting `json:"drafting,omitempty"` - Review string `json:"review,omitempty"` - Source string `json:"source,omitempty"` - Reserved string `json:"reserved,omitempty"` - NeedsOperat bool `json:"needs-operator,omitempty"` + Retracted []RetractionRow `json:"retracted,omitempty"` + Retracting string `json:"retracting,omitempty"` + Reviews []Review `json:"reviews,omitempty"` + // PendingReviews are the reviews of the pending version, with their findings, when get is asked without a + // version (issue 1970): the version waits for its author, and these are what it must answer. + PendingReviews []Review `json:"pending-reviews,omitempty"` + Drafting *Drafting `json:"drafting,omitempty"` + Review string `json:"review,omitempty"` + Source string `json:"source,omitempty"` + Reserved string `json:"reserved,omitempty"` + NeedsOperat bool `json:"needs-operator,omitempty"` // Brief is what the operator's ask shows of it (issue 639). Brief *Brief `json:"brief,omitempty"` Cut string `json:"cut,omitempty"` @@ -371,7 +374,17 @@ func (k *KB) Get(a GetAsk) (*Got, error) { g.Idea = "an idea: research it again before anything rests on it" } if e.Pending != nil && a.Version == 0 { + g.PendingReviews = e.Pending.Reviews g.Pending = fmt.Sprintf("version %d, a progressive insight or a new version of an active effort, waits for its review: history shows what it changes", e.PendingVersion) + if n := len(e.Pending.Reviews); n > 0 && e.Pending.Reviews[n-1].Verdict == VerdictChanges { + last := e.Pending.Reviews[n-1] + k := len(strings.Split(strings.TrimSpace(last.Findings), "\n")) + if strings.TrimSpace(last.Findings) == "" { + k = 0 + } + g.Pending = fmt.Sprintf("version %d: changes asked (%d finding(s)) on version %d, shown in pending-reviews; it waits for its author's revision", + e.PendingVersion, k, last.Version) + } } if r.Ask != nil { g.Waiting = "the operator's ask: " + r.Ask.Said @@ -452,6 +465,10 @@ func (k *KB) Get(a GetAsk) (*Got, error) { cut = append(cut, fmt.Sprintf("%d earlier review(s) are not shown: history lists every version", len(g.Reviews)-1)) g.Reviews = g.Reviews[len(g.Reviews)-1:] } + if size(g) > room && len(g.PendingReviews) > 1 { + cut = append(cut, fmt.Sprintf("%d earlier pending review(s) are not shown: get with the pending version's number shows them", len(g.PendingReviews)-1)) + g.PendingReviews = g.PendingReviews[len(g.PendingReviews)-1:] + } for _, which := range []string{"linked-from", "links"} { if size(g) <= room { break diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/main.go b/modules/mesh-knowledge/cmd/mesh-knowledge/main.go index 5882e6af3..38c37131c 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/main.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/main.go @@ -783,7 +783,7 @@ func Tools(k *KB, caller func(map[string]any) string) []stdio.Tool { }}, {Name: Seat + ".get", Description: "One record by its number: its title, status, topic, date and deciders; its clauses, each with whether it holds; " + - "its links both ways; what changed it since; its versions; its reviews, and the operator's ask when it waits for one. " + + "its links both ways; what changed it since; its versions; its reviews, and the findings of the pending version's reviews (pending-reviews) when one waits for its author; the operator's ask when it waits for one. " + "Its reasoning (context, options, consequences, how it is checked; a research effort's documents) only with reasoning, " + "or one section by name; an earlier version with version. Within a bound it states, 4 KiB unless asked for more, " + "saying what it cut.", diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go index 51221ab22..3d6e79d4d 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go @@ -361,6 +361,15 @@ func TestAnAcceptedRecordChangesOnlyByAnInsightReviewedLikeAProposal(t *testing. e.Pending.Reviews[0].Verdict != VerdictChanges { t.Fatalf("the insight was served before its review approved it: %+v", e.Current.Sections) } + // 1970/AC2: the same for a pending insight. + g, err := w.k.Get(GetAsk{Kind: KindDecision, Number: n}) + if err != nil { + t.Fatal(err) + } + if len(g.PendingReviews) != 1 || !strings.Contains(g.PendingReviews[0].Findings, "what a minute is measured from") || + !strings.Contains(g.Pending, "changes asked") { + t.Fatalf("pending insight findings not shown: %+v, %q", g.PendingReviews, g.Pending) + } w.judge.review = "" p.Sections[2].Text = "Merges land sooner, by about a minute from the last merge." if _, err := w.k.Propose("author@g14", p); err != nil { From b6d85847f8555bb2ee7d69834636d6391ef074f3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 21:51:44 +0100 Subject: [PATCH 02/10] operator-channel.asks answers the open asks within a default bound, and says what it cut (issue 1974) Why: with no argument the verb answered every ask of the month, 737,934 characters, more than a session reads. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/messenger/cmd/messenger/asks.go | 26 +++++++++++++++++--- modules/messenger/cmd/messenger/held_test.go | 17 ++++++------- modules/messenger/cmd/messenger/main.go | 14 +++++------ 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 86ac84332..b8211aacd 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -1418,27 +1418,35 @@ func (h *Holder) AsksNow() []AskView { } // AsksQuery narrows what the tool `operator-channel.asks` answers (issue 595): Key one ask, State open (open and held) or -// ended, Since the asks opened at or after that time, Bound the most bytes of the answer's asks. The zero query keeps the -// whole answer, as before. +// ended (all takes both), Since the asks opened at or after that time, Bound the most bytes of the answer's asks. With no +// argument at all the query is the open asks within DefaultAsksBound (issue 1974), and the answer says so. type AsksQuery struct { Key, State string Since time.Time Bound int + // Defaulted is set by AsksQueryOf when no key, state or since was given: the open asks only. + Defaulted bool } +// DefaultAsksBound is the most bytes of asks an answer holds when the caller gives no bound (issue 1974: the whole of a +// month's asks was 737,934 characters, more than a session reads). +const DefaultAsksBound = 32768 + // AsksAnswer is the answer of a narrowed query: the asks kept, how many the filters matched, and, when the bound cut the // answer short, that it did and how. type AsksAnswer struct { Asks []AskView `json:"asks"` Matched int `json:"matched"` Cut string `json:"cut,omitempty"` + // Default says that no key, state or since was given, so only the open asks are answered, and how to ask for more. + Default string `json:"default,omitempty"` } // AsksQueryOf reads the arguments of the tool; an empty set is the zero query. func AsksQueryOf(a map[string]any) (AsksQuery, error) { q := AsksQuery{Key: strArg(a, "key"), State: strArg(a, "state")} - if q.State != "" && q.State != "open" && q.State != "ended" { - return q, errorf("state is open or ended, not %q", q.State) + if q.State != "" && q.State != "open" && q.State != "ended" && q.State != "all" { + return q, errorf("state is open, ended or all, not %q", q.State) } if since := strArg(a, "since"); since != "" { t, err := time.Parse(time.RFC3339, since) @@ -1453,6 +1461,12 @@ func AsksQueryOf(a map[string]any) (AsksQuery, error) { } q.Bound = int(v) } + if q.Key == "" && q.State == "" && q.Since.IsZero() { + q.State, q.Defaulted = "open", true + } + if q.Bound == 0 { + q.Bound = DefaultAsksBound + } return q, nil } @@ -1461,6 +1475,10 @@ func AsksQueryOf(a map[string]any) (AsksQuery, error) { func (q AsksQuery) Narrow(all []AskView) AsksAnswer { var out AsksAnswer out.Asks = []AskView{} + if q.Defaulted { + out.Default = "no key, state or since was given, so only the open asks are shown: state ended or all for the ended ones, " + + "key for one ask, since for the asks opened from a time" + } var kept []AskView for _, v := range all { open := v.State == StateOpen || v.State == StateHeld diff --git a/modules/messenger/cmd/messenger/held_test.go b/modules/messenger/cmd/messenger/held_test.go index 2d6c74c1e..dd545eee7 100644 --- a/modules/messenger/cmd/messenger/held_test.go +++ b/modules/messenger/cmd/messenger/held_test.go @@ -143,7 +143,7 @@ func TestAHeldAskNamesEveryKindAndWhyWhenNoChannelIsReady(t *testing.T) { } // Issue 595: operator-channel.asks answers one ask, the open or the ended, those opened from a time, and no more bytes than a -// bound, said when it cut the answer short; with no argument it keeps the whole answer. +// bound, said when it cut the answer short; with no argument it answers the open asks only, within a default bound, and says so (issue 1974). func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) { view := func(key, state, opened string) AskView { return AskView{Key: key, Asker: "mesh-delivery", Headline: "Approve " + key, State: state, Opened: opened} @@ -159,13 +159,15 @@ func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) { cut bool err bool }{ - {"no argument is the zero query", map[string]any{}, nil, 0, false, false}, + {"no argument is the open asks, said", map[string]any{}, []string{"a.3", "a.2"}, 2, false, false}, + {"state all is every ask", map[string]any{"state": "all"}, []string{"a.3", "a.2", "a.1", "a.0"}, 4, false, false}, + {"a key finds an ended ask whatever its state", map[string]any{"key": "a.0"}, []string{"a.0"}, 1, false, false}, {"key", map[string]any{"key": "a.1"}, []string{"a.1"}, 1, false, false}, {"state open takes open and held", map[string]any{"state": "open"}, []string{"a.3", "a.2"}, 2, false, false}, {"state ended", map[string]any{"state": "ended"}, []string{"a.1", "a.0"}, 2, false, false}, {"since", map[string]any{"since": "2026-11-02T12:00:00Z"}, []string{"a.3", "a.2"}, 2, false, false}, {"bound cuts and says so", map[string]any{"bound": float64(2*len(one) + 10), "state": "open"}, []string{"a.3", "a.2"}, 2, false, false}, - {"a bound under the asks cuts", map[string]any{"bound": float64(len(one) + 5)}, []string{"a.3"}, 4, true, false}, + {"a bound under the asks cuts", map[string]any{"bound": float64(len(one) + 5)}, []string{"a.3"}, 2, true, false}, {"a state that is neither", map[string]any{"state": "done"}, nil, 0, false, true}, {"a time that is not one", map[string]any{"since": "yesterday"}, nil, 0, false, true}, } { @@ -177,13 +179,10 @@ func TestTheAsksVerbNarrowsItsAnswer(t *testing.T) { if tc.err { continue } - if tc.want == nil { - if q != (AsksQuery{}) { - t.Errorf("%s: %+v is not the zero query, which keeps today's answer", tc.name, q) - } - continue - } got := q.Narrow(all) + if (got.Default != "") != (tc.args["key"] == nil && tc.args["state"] == nil && tc.args["since"] == nil) || q.Bound == 0 { + t.Errorf("%s: default %q, bound %d: only no argument is defaulted, and every query is bounded", tc.name, got.Default, q.Bound) + } var keys []string for _, v := range got.Asks { keys = append(keys, v.Key) diff --git a/modules/messenger/cmd/messenger/main.go b/modules/messenger/cmd/messenger/main.go index d1346ef97..ad4946aa8 100644 --- a/modules/messenger/cmd/messenger/main.go +++ b/modules/messenger/cmd/messenger/main.go @@ -734,12 +734,13 @@ func tools(h *Holder, l *listening) []stdio.Tool { {Name: "operator-channel.asks", Description: "The asks this router holds (novox/hq ADR 0259): open (or held, with since when and why no channel can carry it yet) first, then the last month's — each " + "one's asker, headline, state, when it opened and expires, the channels it was offered on, how it " + - "ended (the warrant: who chose what, through which channel), and whether its asker was told. With no argument every " + - "ask, which is large; filter it: key for one ask, state open or ended, since (a time, RFC 3339) for the asks opened " + - "from then, and bound, the most bytes of the answer, which says when it cut it short.", + "ended (the warrant: who chose what, through which channel), and whether its asker was told. With no argument " + + "only the open asks, within 32768 bytes; the answer says what it cut and how to ask for more: key for one ask " + + "(any state), state open, ended or all, since (a time, RFC 3339) for the asks opened from then, and bound, the " + + "most bytes of the answer.", Input: map[string]any{ - "key": str("one ask, by its key (asker.id); none for every ask"), - "state": str("open (open and held) or ended; none for both"), + "key": str("one ask, by its key (asker.id); none for the open asks"), + "state": str("open (open and held), ended or all; none is open when no key or since is given, else both"), "since": str("only the asks opened at or after this time, RFC 3339"), "bound": map[string]any{"type": "integer", "description": "the most bytes of the asks in the answer; said when it cuts the answer short"}}, Run: func(a map[string]any) (any, error) { @@ -747,9 +748,6 @@ func tools(h *Holder, l *listening) []stdio.Tool { if err != nil { return nil, err } - if q == (AsksQuery{}) { - return h.AsksNow(), nil - } return q.Narrow(h.AsksNow()), nil }}, {Name: "messenger_identities", From 0220dfa92b4f4f0f0566302b79e53a0de0194886 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 22:03:51 +0100 Subject: [PATCH 03/10] Answer an ask the build module cannot read on its step key as refused (issue 1921, seams must 1) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-go/buildstep/pull.go | 37 ++++++++++++++++++-- modules/mesh-build-go/buildstep/pull_test.go | 33 +++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/buildstep/pull.go b/modules/mesh-build-go/buildstep/pull.go index b7e54ac6d..9f5ed04b4 100644 --- a/modules/mesh-build-go/buildstep/pull.go +++ b/modules/mesh-build-go/buildstep/pull.go @@ -561,6 +561,36 @@ func (p *Puller) declineRun(msg *Pulled, ask delivery.StepAsk, key string, step p.mu.Unlock() } +// refuseUnread ends an ask this module cannot read: when its data holds an id that keys a step, the step is written +// refused with the reason, so the asker learns at once that no node began it; otherwise (no id, not JSON) the ask is only +// dropped and logged. The message is dropped either way. +func (p *Puller) refuseUnread(msg *Pulled, why error) { + var probe struct { + ID string `json:"id"` + Kind string `json:"kind"` + } + if json.Unmarshal(msg.Data, &probe) == nil && probe.ID != "" { + if key, kerr := delivery.AskStepKey(p.Node, probe.ID); kerr == nil { + tool := "ask" + if probe.Kind != "" { + tool = probe.Kind + " ask" + } + at := now().UTC() + step := delivery.Step{ID: probe.ID, Tool: tool, Node: p.Node, State: delivery.StepRefused, Started: at, Ended: at, + Reason: fmt.Sprintf("this build module on %s cannot read the ask: %v", p.Node, why), Owner: sdkstep.Owner} + if werr := writeEnd(p.Put, key, step); werr != nil { + p.logf("the unread ask %s cannot be answered on %s: %v", probe.ID, p.Node, werr) + } else { + p.logf("an ask refused: %v", why) + _ = msg.Term() + return + } + } + } + p.logf("an ask dropped: %v", why) + _ = msg.Term() +} + func withKind(msg *Pulled, kind string) *Pulled { if msg != nil { msg.Kind = kind @@ -576,9 +606,10 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { err = fmt.Errorf("the ask %s is a %s ask of %s, pulled from the %s %s worker", ask.ID, ask.Kind, ask.Language, p.Language, kind) } if err != nil { - // An ask that is not one runs nowhere: delivering it again would only fail again. - p.logf("an ask dropped: %v", err) - _ = msg.Term() + // An ask that is not one runs nowhere: delivering it again would only fail again. But an ask that names an id + // is answered on its step key as refused, naming why (a job this build module does not know, a version it does + // not read): a dropped ask must always be answered, or its asker waits to its bound for nothing. + p.refuseUnread(msg, err) return } if lacks := delivery.Lacks(ask.Needs, p.Has); len(lacks) > 0 { diff --git a/modules/mesh-build-go/buildstep/pull_test.go b/modules/mesh-build-go/buildstep/pull_test.go index 086b15527..d196d8eec 100644 --- a/modules/mesh-build-go/buildstep/pull_test.go +++ b/modules/mesh-build-go/buildstep/pull_test.go @@ -519,3 +519,36 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) { t.Errorf("acked %v naked %v, want it termed alone", acked, naked) } } + +// An ask whose job this build module does not know is answered on its step key as refused, naming why, never only dropped +// (a dropped ask leaves its asker waiting to its bound); one with no id is still dropped. +func TestPullAnswersAnAskItCannotRead(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + good := testAsk(delivery.AskCheck) + raw, err := delivery.EncodeStepAsk(good) + if err != nil { + t.Fatal(err) + } + var m map[string]any + _ = json.Unmarshal(raw, &m) + delete(m, "check") + m["frobnicate"] = map[string]any{"program": "x"} + odd, _ := json.Marshal(m) + // The job is not one this reader holds, so the ask carries none it knows: decode refuses it. + if _, err := delivery.DecodeStepAsk(odd); err == nil { + t.Fatal("the fixture ask decodes") + } + src.Publish(delivery.AskCheck, odd) + src.Publish(delivery.AskCheck, []byte(`{"v":1}`)) + p := &Puller{Source: src, Language: delivery.AskGo, Node: "ace", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { t.Error("an unread ask ran"); return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "both termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 }) + key, _ := delivery.AskStepKey("ace", good.ID) + s, ok := st.get(key) + if !ok || s.State != delivery.StepRefused || !strings.Contains(s.Reason, "cannot read the ask") || !strings.Contains(s.Reason, "exactly one") { + t.Fatalf("the unread ask's step = %+v (%v)", s, ok) + } +} From 428ef8bbe58503add41fc0d1ab3ff720cf0e7f47 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 22:05:05 +0100 Subject: [PATCH 04/10] Kept composers and programs get a list tool, a drop tool and a sweep (issue 1921, seams must 2) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-go/README.md | 4 + .../cmd/mesh-build-go/compose.go | 9 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 1 + .../mesh-build-go/cmd/mesh-build-go/kept.go | 130 ++++++++++++++++++ .../cmd/mesh-build-go/kept_test.go | 71 ++++++++++ .../cmd/mesh-build-go/module_test.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/tools.go | 1 + modules/mesh-build-go/module.json | 2 + 8 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/kept.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/kept_test.go diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index ff16837f8..ce1d97989 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -49,6 +49,10 @@ already gone is no error, one a running step holds is refused); its caller calls failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR 0509), and the removal is said in the log as the caller's fault. +The programs this module builds from a commit and keeps in its cache (composers of compose asks, checking programs of run +asks) are read by `build_go_kept` (kind, key, size, last use) and removed by `build_go_kept_drop {kind, key}`; one no ask used +for 30 days is swept with the workspaces (issue 1921). + A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's `pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go index 1d55fbf9d..a8f52804f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -38,7 +38,7 @@ var composerLocks sync.Map // composerPath is where the composer of key is kept: the cache directory, beside Go's own caches. func (m *Module) composerPath(key string) string { - return filepath.Join(m.Cache, "composers", key, "mesh-compose") + return m.programPath("composers", key) } // compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled. @@ -99,10 +99,11 @@ func (m *Module) manifestOf(digest string) ([]byte, error) { // composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true). func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) { exe := m.composerPath(key) - mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{}) - mu.(*sync.Mutex).Lock() - defer mu.(*sync.Mutex).Unlock() + mu := lockOf(exe) + mu.Lock() + defer mu.Unlock() if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() { + touch(exe, m.clock()) return exe, false, nil } if fetch == nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b733df269..a1988224a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -348,6 +348,7 @@ func (m *Module) goEnvFor(dir string) []string { // net for a caller that never released one (issue 1839, ADR 0509). func (m *Module) CleanWorkspaces(now time.Time) { m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) + m.SweepKept(now, KeptBound) } // clock is the time now: Now when a test set it, else the wall clock. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept.go b/modules/mesh-build-go/cmd/mesh-build-go/kept.go new file mode 100644 index 000000000..7e6bc2344 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept.go @@ -0,0 +1,130 @@ +package main + +// **Kept programs have tools and a sweep** (novox/hq issue 1921, the seams review's must 2): the programs this module builds +// from a commit and keeps under its cache so each commit is built once on a node, the composers of compose asks and the +// programs of run asks, are read by build_go_kept, removed one by one by build_go_kept_drop, and swept when unused for +// KeptBound. A use touches the program's file, so its modification time is its last use. + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The tools of the kept programs. +const ( + KeptTool = "build_go_kept" + KeptDropTool = "build_go_kept_drop" +) + +// KeptBound is how long a kept program that no ask used is kept: a month, longer than any group lives, so only the build +// of a commit nobody asks about any more goes. +const KeptBound = 30 * 24 * time.Hour + +// keptKinds are the cache directories of kept programs and the name of the program in each key's directory. +var keptKinds = map[string]string{"composers": "mesh-compose", "programs": "program"} + +// KeptProgram is one kept program: its kind (the cache directory), its key, its size and when an ask last used it. +type KeptProgram struct { + Kind string `json:"kind"` + Key string `json:"key"` + Bytes int64 `json:"bytes"` + LastUsed time.Time `json:"last-used"` +} + +// programPath is where the program of kind and key is kept. +func (m *Module) programPath(kind, key string) string { + return filepath.Join(m.Cache, kind, key, keptKinds[kind]) +} + +// lockOf is the mutex that guards the program at path against a build or a use while it is dropped. +func lockOf(path string) *sync.Mutex { + l, _ := composerLocks.LoadOrStore(path, &sync.Mutex{}) + return l.(*sync.Mutex) +} + +// touch records a use of the kept program at path. +func touch(path string, at time.Time) { _ = os.Chtimes(path, at, at) } + +// Kept lists the kept programs, composers then programs, each kind by key. +func (m *Module) Kept() []KeptProgram { + out := []KeptProgram{} + for _, kind := range []string{"composers", "programs"} { + entries, _ := os.ReadDir(filepath.Join(m.Cache, kind)) + for _, e := range entries { + if !e.IsDir() { + continue + } + st, err := os.Stat(m.programPath(kind, e.Name())) + if err != nil || !st.Mode().IsRegular() { + continue + } + out = append(out, KeptProgram{Kind: kind, Key: e.Name(), Bytes: st.Size(), LastUsed: st.ModTime().UTC()}) + } + } + return out +} + +// DropKept removes the kept program of kind and key; one already gone is no error. +func (m *Module) DropKept(kind, key string) error { + if _, ok := keptKinds[kind]; !ok { + return fmt.Errorf("a kept program is of kind composers or programs, not %q", kind) + } + if key == "" || key != filepath.Base(key) || key == "." || key == ".." { + return fmt.Errorf("%q is no kept program's key", key) + } + exe := m.programPath(kind, key) + l := lockOf(exe) + l.Lock() + defer l.Unlock() + return os.RemoveAll(filepath.Dir(exe)) +} + +// SweepKept removes the kept programs no ask used within bound before now, and answers what it removed. +func (m *Module) SweepKept(now time.Time, bound time.Duration) []KeptProgram { + var gone []KeptProgram + for _, k := range m.Kept() { + if now.Sub(k.LastUsed) <= bound { + continue + } + if m.DropKept(k.Kind, k.Key) == nil { + gone = append(gone, k) + } + } + sort.Slice(gone, func(i, j int) bool { return gone[i].Key < gone[j].Key }) + if len(gone) > 0 && m.Say != nil { + m.Say("removed %d kept programs no ask used within %s", len(gone), bound) + } + return gone +} + +// keptTools are the tools that read and drop the kept programs: no guard and no floor, since they only free room. +func (m *Module) keptTools() []stdio.Tool { + return []stdio.Tool{ + {Name: KeptTool, + Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " + + "the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " + + "for 30 days is removed. Read-only.", + Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }}, + {Name: KeptDropTool, + Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " + + "for that commit builds it again. One already gone is no error.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}}, + "key": map[string]any{"type": "string", "description": "the key build_go_kept names"}, + }, "required": []string{"kind", "key"}}, + Run: func(args map[string]any) (any, error) { + kind, _ := args["kind"].(string) + key, _ := args["key"].(string) + if err := m.DropKept(kind, key); err != nil { + return nil, err + } + return map[string]any{"dropped": key, "kind": kind}, nil + }}, + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go new file mode 100644 index 000000000..615f8b36a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func keepProgram(t *testing.T, m *Module, kind, key string, size int, used time.Time) { + t.Helper() + exe := m.programPath(kind, key) + if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(exe, make([]byte, size), 0o700); err != nil { + t.Fatal(err) + } + touch(exe, used) +} + +func toolOf(t *testing.T, m *Module, name string) func(map[string]any) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == name { + return tool.Run + } + } + t.Fatalf("no tool %s", name) + return nil +} + +// Seams review must 2 (issue 1921): the kept composers and programs are listed with their size and last use, dropped one +// by one, and swept when unused past KeptBound; a use keeps a program. +func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) { + m, _ := fetchModule(t, &fakeStep{}) + now := time.Now().UTC().Truncate(time.Second) + keepProgram(t, m, "composers", "old", 10, now.Add(-KeptBound-time.Hour)) + keepProgram(t, m, "programs", "recent", 20, now.Add(-time.Hour)) + keepProgram(t, m, "programs", "gone", 30, now) + + out, err := toolOf(t, m, KeptTool)(nil) + if err != nil { + t.Fatal(err) + } + list := out.(map[string]any)["kept"].([]KeptProgram) + if len(list) != 3 || list[0].Kind != "composers" || list[0].Key != "old" || list[0].Bytes != 10 || !list[0].LastUsed.Equal(now.Add(-KeptBound-time.Hour)) { + t.Fatalf("the kept programs = %+v", list) + } + + for _, bad := range []map[string]any{{"kind": "nope", "key": "gone"}, {"kind": "programs", "key": "../composers"}, {"kind": "programs", "key": ""}} { + if _, err := toolOf(t, m, KeptDropTool)(bad); err == nil { + t.Errorf("dropped with %v", bad) + } + } + for i := 0; i < 2; i++ { // a second drop is no error + if _, err := toolOf(t, m, KeptDropTool)(map[string]any{"kind": "programs", "key": "gone"}); err != nil { + t.Fatalf("drop %d: %v", i, err) + } + } + if _, err := os.Stat(m.programPath("programs", "gone")); !os.IsNotExist(err) { + t.Error("the dropped program is still kept") + } + + if swept := m.SweepKept(now, KeptBound); len(swept) != 1 || swept[0].Key != "old" { + t.Fatalf("swept %+v, want the old composer only", swept) + } + if got := m.Kept(); len(got) != 1 || got[0].Key != "recent" { + t.Fatalf("kept after the sweep = %+v", got) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go index a780172fd..eba3f02cc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go @@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { {Name: "short_probe", Run: run}, }, m.StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool { + if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool { continue } var reason string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index dbc20eab0..de6bc5a65 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, m.ReadTools()...) tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held)) tools = append(tools, m.releaseTool()) + tools = append(tools, m.keptTools()...) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " + diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 2c6289c6a..ee94e020f 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -50,6 +50,8 @@ "build_go_build", "build_go_imports", "build_go_release", + "build_go_kept", + "build_go_kept_drop", "build_go_step", "build_go_steps", "build_go_step_end", From fa73674e22023a3803dd3364a7f2e7e1db796b99 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 22:20:25 +0100 Subject: [PATCH 05/10] A run ask builds a checking program once and runs it over its heads and input; compose keeps seats (issue 1921, C1-C3) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-go/buildstep/pull_test.go | 5 +- .../mesh-build-go/cmd/mesh-build-go/asks.go | 3 + .../cmd/mesh-build-go/compose.go | 27 ++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 1 + .../mesh-build-go/cmd/mesh-build-go/module.go | 6 +- .../mesh-build-go/cmd/mesh-build-go/run.go | 136 ++++++++++++++++++ .../cmd/mesh-build-go/run_integration_test.go | 105 ++++++++++++++ .../cmd/mesh-build-go/run_test.go | 106 ++++++++++++++ modules/mesh-build-go/module.json | 1 + 9 files changed, 378 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/run.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/run_test.go diff --git a/modules/mesh-build-go/buildstep/pull_test.go b/modules/mesh-build-go/buildstep/pull_test.go index d196d8eec..9a33b5a4c 100644 --- a/modules/mesh-build-go/buildstep/pull_test.go +++ b/modules/mesh-build-go/buildstep/pull_test.go @@ -541,7 +541,10 @@ func TestPullAnswersAnAskItCannotRead(t *testing.T) { src.Publish(delivery.AskCheck, odd) src.Publish(delivery.AskCheck, []byte(`{"v":1}`)) p := &Puller{Source: src, Language: delivery.AskGo, Node: "ace", Put: st.put, Capacity: func() int { return 1 }, - PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { t.Error("an unread ask ran"); return nil, nil }} + PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { + t.Error("an unread ask ran") + return nil, nil + }} ctx, cancel := context.WithCancel(context.Background()) defer cancel() go p.Pull(ctx) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 5579cc6c8..3ddcb7c43 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -121,6 +121,9 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep if a.Compose != nil { return m.compose(ctx, ask.Fetch, a) } + if a.Run != nil { + return m.run(ctx, ask.Fetch, a) + } return buildstep.RunAsk(ctx, ask, a) }} } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go index a8f52804f..08688be00 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -98,7 +98,14 @@ func (m *Module) manifestOf(digest string) ([]byte, error) { // composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true). func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) { - exe := m.composerPath(key) + return m.programOf(ctx, fetch, "composers", "composer", key, job.Fetch, job.Dir, job.Package) +} + +// programOf is the program kept under key in the cache directory kind, or, when none is, the one built now: the commit +// fetch names is fetched (the fetch step: the forge key and the network, none of the change's code), and package in the +// Go module of dir is built in a run step shut off with nothing fetched. what names it in a failure. +func (m *Module) programOf(ctx context.Context, fetch buildstep.StepFunc, kind, what, key string, fetchArgs map[string]any, dir, pkg string) (string, bool, error) { + exe := m.programPath(kind, key) mu := lockOf(exe) mu.Lock() defer mu.Unlock() @@ -109,7 +116,7 @@ func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job d if fetch == nil { return "", false, buildstep.Refuse("this build module fetches nothing") } - out, err := fetch(ctx, job.Fetch) + out, err := fetch(ctx, fetchArgs) if err != nil { return "", false, err } @@ -126,11 +133,11 @@ func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job d defer m.releaseSaid(f.Workspace) done := m.spaces().Hold(f.Workspace) defer done() - dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir) - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { - return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir) + tree := filepath.Join(m.Work, f.Workspace, "tree", dir) + if _, err := os.Stat(filepath.Join(tree, "go.mod")); err != nil { + return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no %s can be built there", fetchArgs["repository"], fetchArgs["commit"], dir, what) } - into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose") + into := filepath.Join(m.Work, f.Workspace, what, "built") if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil { return "", false, err } @@ -138,15 +145,15 @@ func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job d if err != nil { return "", false, err } - env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0") - r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package) + env := append(m.goEnvFor(tree), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0") + r := run(ctx, env, "go", "-C", tree, "build", "-trimpath", "-o", into, pkg) if r.Status != 0 || r.Err != "" { // The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole, // so the asker tells that apart from a fault. - return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + return "", false, fmt.Errorf("the %s %s does not build at %s: %s", what, pkg, fetchArgs["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) } if err := copyExecutable(into, exe); err != nil { - return "", false, fmt.Errorf("the composer could not be kept: %w", err) + return "", false, fmt.Errorf("the %s could not be kept: %w", what, err) } return exe, true, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 4e96528b5..03a21b9af 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -58,6 +58,7 @@ func main() { m.Settings = m.Home + "/settings.json" m.ReadKept = readOf(stdio.State(KeptState)).Get m.ReadManifest = readOf(stdio.State(ManifestsState)).Get + m.ReadInput = readOf(stdio.State(InputsState)).Get m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap // Outside a launch by the runtime — a build, a check — there is no node and no state to write. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 38e2f843c..6cf40a48e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -93,7 +93,11 @@ type Module struct { // ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request // names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test). ReadManifest func(key string) (json.RawMessage, error) - GoModCache string + // ReadInput reads a key of the checker's inputs state (InputsState), which a run ask's input names by its digest. + ReadInput func(key string) (json.RawMessage, error) + // PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test). + PlaceHead func(ctx context.Context, h delivery.RunHead, dir string) error + GoModCache string spacesOnce sync.Once workspaces *buildstep.Workspaces diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go new file mode 100644 index 000000000..ce14ea74a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -0,0 +1,136 @@ +package main + +// **A run ask runs a program built from a change, over heads it reads** (novox/hq ADR 0644, ADR 0656; issue 1921): the +// checking program of a group's check. The module builds the program at the commit the ask names, once per +// delivery.RunKey, as a compose ask builds a composer (programOf); fetches every head the ask names and lays each read-only +// under the directory MESH_RUN_HEADS names, at `/`; reads the input the ask names by its digest from the +// checker's inputs state and writes it to the file MESH_RUN_INPUT names; and runs the program shut off with the ask's +// arguments: no network, none of the account. The answer is delivery.RunAnswer: the program's exit, the last line of its +// standard output and the first line of its standard error. A program that fails is an answer, not a fault of the step. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// InputsState is the checker's state that keeps a run's input by its digest (the module reads it, module.json reads). +const InputsState = delivery.CheckerModule + ".inputs" + +// run runs a run ask: the program of its commit, built once, over its heads and its input. +func (m *Module) run(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) { + job := *a.Run + if job.Input.State != InputsState { + return nil, buildstep.Refuse("the run ask %s keeps its input in %s; this module reads %s", a.ID, job.Input.State, InputsState) + } + body, err := m.inputOf(job.Input) + if err != nil { + return nil, err + } + installed, err := m.Probe(ctx) + if err != nil { + return nil, err + } + key := delivery.RunKey(job.Program, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, a.Packages) + fetchArgs := sdkstep.FetchRequest{Repository: job.Program.Repository, Commit: job.Program.Commit, Packages: a.Packages}.Args() + exe, built, err := m.programOf(ctx, fetch, "programs", "checking program", key, fetchArgs, job.Program.Dir, job.Program.Package) + if err != nil { + return nil, err + } + ans, err := m.runProgram(ctx, exe, job, body) + if err != nil { + return nil, err + } + ans.Built, ans.Key = built, key + return ans, nil +} + +// inputOf is the input a run names, read from the checker's inputs state and checked against its digest and size. +func (m *Module) inputOf(in delivery.RunInput) ([]byte, error) { + if m.ReadInput == nil { + return nil, errors.New("this module reads no inputs state") + } + raw, err := m.ReadInput(in.Digest) + if err != nil { + return nil, err + } + if raw == nil { + return nil, fmt.Errorf("the checker keeps no input of digest %s", in.Digest) + } + if err := in.Verify(raw); err != nil { + return nil, buildstep.Refuse("%v", err) + } + return raw, nil +} + +// placeHead lays one head's commit as a plain directory at dir, with no symlink in it; a test sets its own. +func (m *Module) placeHead(ctx context.Context, h delivery.RunHead, dir string) error { + if m.PlaceHead != nil { + return m.PlaceHead(ctx, h, dir) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(h.Repository), h.Commit, "", m.fetchOptions(ctx, h.Repository, h.Commit, "")...); err != nil { + return err + } + if err := m.dissociate(ctx, dir); err != nil { + return err + } + if link := firstSymlink(dir); link != "" { + return fmt.Errorf("%s holds a symlink, %s", h.Repository, link) + } + return nil +} + +// runProgram lays the heads and the input in a workspace of its own and runs exe shut off over them. +func (m *Module) runProgram(ctx context.Context, exe string, job delivery.RunJob, input []byte) (delivery.RunAnswer, error) { + id := sdkstep.NewID() + done := m.spaces().Hold(id) + defer m.releaseSaid(id) + defer done() + ws := filepath.Join(m.Work, id) + heads := filepath.Join(ws, "heads") + for _, h := range job.Heads { + if err := m.placeHead(ctx, h, filepath.Join(heads, filepath.FromSlash(h.Repository))); err != nil { + return delivery.RunAnswer{}, fmt.Errorf("the head of %s at %s: %w", h.Repository, h.Commit, err) + } + } + if err := os.MkdirAll(heads, 0o700); err != nil { + return delivery.RunAnswer{}, err + } + file := filepath.Join(ws, "input") + if err := os.WriteFile(file, input, 0o600); err != nil { + return delivery.RunAnswer{}, err + } + run := m.Run + if !m.Inside { + iso := m.isolation(id, false) + iso.ReadOnly = append(iso.ReadOnly, exe) + run = buildstep.Isolated(m.Run, iso) + } + // The heads are bound read-only inside the writable workspace by running the program with them as its read-only + // directory too (a later bind of the same path wins). + r := run(ctx, []string{delivery.RunHeadsEnv + "=" + heads, delivery.RunInputEnv + "=" + file}, exe, job.Args...) + if r.Err != "" && r.Status == 0 { + return delivery.RunAnswer{}, fmt.Errorf("the checking program could not run: %s", r.Err) + } + ans := delivery.RunAnswer{Exit: r.Status, Said: firstLine(r.Err + "\n" + r.Stderr)} + line := strings.TrimSpace(lastLine(r.Stdout)) + switch { + case line == "": + case json.Valid([]byte(line)): + ans.Output = json.RawMessage(line) + default: + ans.Output, _ = json.Marshal(line) + } + return ans, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go new file mode 100644 index 000000000..c1463d2a8 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go @@ -0,0 +1,105 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// 1921/C3: a run ask put on the Go build seat's check stream is pulled by the module started whole on the real bus and +// answered on its step key .: the program, run shut off over its heads and its input (read from the +// checker's inputs state by digest), answers its last line and exit; a program that dials the machine's network is +// shut off, and the answer says it failed. +func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { + needTools(t, "the shut-off run", "bash", "bwrap") + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 2*time.Minute); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") }) + tcp, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer tcp.Close() + + states := newStates() + m := moduleWith(states, "go1.26.0") + m.Work, m.Cache, m.Home = t.TempDir(), t.TempDir(), t.TempDir() + m.Capacity = func() int { return 1 } + m.Run = StepRunner(m.Home) + for _, c := range m.caches() { + if err := os.MkdirAll(c, 0o700); err != nil { + t.Fatal(err) + } + } + m.PlaceHead = func(_ context.Context, h delivery.RunHead, dir string) error { return os.MkdirAll(dir, 0o700) } + body := `{"v":1}` + m.ReadInput = func(string) (json.RawMessage, error) { return json.RawMessage(body), nil } + + a := runAskOf(t, body) + key := delivery.RunKey(a.Run.Program, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}, nil) + exe := m.programPath("programs", key) + os.MkdirAll(filepath.Dir(exe), 0o700) + script := "#!/bin/bash\n" + `echo "start"; exec 3<>/dev/tcp/127.0.0.1/` + strings.Split(tcp.Addr().String(), ":")[1] + " || exit 7\n" + `echo '{"dialed":true}'` + "\n" + if err := os.WriteFile(exe, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + p := m.Puller(fakeSteps(new([]string)), &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo}) + p.PullWait = 200 * time.Millisecond + go p.Pull(ctx) + + raw, err := delivery.EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + stepKey, _ := delivery.AskStepKey("ace", a.ID) + var step delivery.Step + for deadline := time.Now().Add(60 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + states.mu.Lock() + v := states.steps[stepKey] + states.mu.Unlock() + if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning { + step = v[len(v)-1] + break + } + } + if step.State != delivery.StepPassed { + t.Fatalf("the run ask ended %q: %s", step.State, step.Reason) + } + var ans delivery.RunAnswer + if err := json.Unmarshal(step.Result, &ans); err != nil { + t.Fatal(err) + } + if ans.Key != key || ans.Exit != 7 || strings.Contains(string(ans.Output), "dialed") || string(ans.Output) != `"start"` { + t.Fatalf("the answer of a program that dials out = %+v output %s", ans, ans.Output) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go new file mode 100644 index 000000000..10baa06a8 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const runCommit = "5555555555555555555555555555555555555555" + +func runAskOf(t *testing.T, body string) delivery.StepAsk { + t.Helper() + in, err := delivery.RunInputOf(InputsState, []byte(body)) + if err != nil { + t.Fatal(err) + } + return delivery.StepAsk{ID: sdkstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-checker", At: time.Now(), + Run: &delivery.RunJob{Program: delivery.RunProgram{Repository: "novox/mesh-delivery", Commit: runCommit, Dir: ".", Package: "./cmd/prog"}, + Heads: []delivery.RunHead{{Repository: "novox/mesh-sdk", Commit: runCommit}, {Repository: "novox/mesh-host", Commit: runCommit}}, + Input: in, Args: []string{"-seats"}}} +} + +// 1921/C1: a kept checking program is run over the input checked against its digest and the heads laid under +// MESH_RUN_HEADS at /, and the answer carries its exit and last line; nothing is fetched or built; a +// program that fails, or says no JSON, is an answer; an input that is not the one named is refused; and the program was +// kept under delivery.RunKey, so a second ask finds it. +func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { + m := composeModule(t) + body := `{"v":1,"case":"x"}` + m.ReadInput = func(key string) (json.RawMessage, error) { + if key != runAskOf(t, body).Run.Input.Digest { + return nil, nil + } + return json.RawMessage(body), nil + } + m.PlaceHead = func(_ context.Context, h delivery.RunHead, dir string) error { + return os.MkdirAll(dir, 0o700) + } + a := runAskOf(t, body) + key := delivery.RunKey(a.Run.Program, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}, nil) + exe := m.programPath("programs", key) + os.MkdirAll(filepath.Dir(exe), 0o700) + script := "#!/bin/sh\n" + + `echo "noise"; echo "args $*" >&2; ` + + `printf '{"input":%s,"heads":"%s"}\n' "$(cat "$MESH_RUN_INPUT")" "$(cd "$MESH_RUN_HEADS" && ls -d */* | tr '\n' ' ')"; exit 3` + "\n" + if err := os.WriteFile(exe, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + fetched := 0 + fetch := func(context.Context, map[string]any) (any, error) { fetched++; return nil, nil } + out, err := m.run(context.Background(), fetch, a) + if err != nil { + t.Fatal(err) + } + ans := out.(delivery.RunAnswer) + if ans.Built || ans.Key != key || fetched != 0 || ans.Exit != 3 || ans.Said != "args -seats" || + string(ans.Output) != `{"input":{"v":1,"case":"x"},"heads":"novox/mesh-host novox/mesh-sdk "}` { + t.Fatalf("answer = %+v (fetched %d) output %s", ans, fetched, ans.Output) + } + if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { + t.Errorf("the run's workspace was left: %v", entries) + } + // An input other than the one named is refused, naming it; an unknown state is refused. + m.ReadInput = func(string) (json.RawMessage, error) { return json.RawMessage(`{"other":1}`), nil } + if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "not the one the ask names") { + t.Errorf("a wrong input: %v", err) + } + a.Run.Input.State = "mesh-delivery-indexer.manifests" + if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "this module reads") { + t.Errorf("another state: %v", err) + } +} + +// 1921/C2 (F2): a compose ask's seats reach the composer: the module decodes and fills the request and writes it again, +// and used to drop the seats. +func TestAComposeAskKeepsItsSeats(t *testing.T) { + m := composeModule(t) + read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`) + m.ReadManifest = read + a := composeAskOf(t, digests[0]) + req := composer.Request{V: composer.Version, Seats: []json.RawMessage{json.RawMessage(`{"seat":"node-x"}`)}, + Cases: []composer.Case{{Name: "change", Manifests: map[string]composer.Manifest{"demo": {Digest: digests[0]}}}}} + a.Compose.Request, _ = json.Marshal(req) + key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}, nil) + exe := m.composerPath(key) + os.MkdirAll(filepath.Dir(exe), 0o700) + script := "#!/bin/sh\n" + `if grep -q '"seats":\[{"seat":"node-x"}\]' "$2"; then S=true; else S=false; fi; ` + + `echo '{"v":1,"sdk":"stand-in","cases":[{"name":"change","manifests":{"demo":{"accepted":'$S'}}}]}'` + "\n" + os.WriteFile(exe, []byte(script), 0o700) + out, err := m.compose(context.Background(), nil, a) + if err != nil { + t.Fatal(err) + } + var v composer.Verdict + if err := json.Unmarshal(out.(delivery.ComposeAnswer).Verdict, &v); err != nil || !v.Cases[0].Manifests["demo"].Accepted { + t.Fatalf("the composer did not receive the seats: %+v %v", v, err) + } +} diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index ee94e020f..3281af8d3 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -20,6 +20,7 @@ } ], "reads": [ + "mesh-delivery-checker.inputs", "mesh-delivery-checker.kept", "mesh-delivery-indexer.manifests" ], From da1356cd7ca2bfab3e702aff04cf97ac71217128 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 23:24:48 +0100 Subject: [PATCH 06/10] Bind a run's heads read-only over its workspace, read its input from the state the ask names, and prove the kept programs' sweep (issue 1921, review round 1) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-go/buildstep/isolate.go | 22 ++++++---- .../cmd/mesh-build-go/kept_test.go | 7 ++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 3 +- .../mesh-build-go/cmd/mesh-build-go/module.go | 4 +- .../mesh-build-go/cmd/mesh-build-go/run.go | 18 +++----- .../cmd/mesh-build-go/run_integration_test.go | 8 ++-- .../cmd/mesh-build-go/run_test.go | 42 +++++++++++++++---- 7 files changed, 64 insertions(+), 40 deletions(-) diff --git a/modules/mesh-build-go/buildstep/isolate.go b/modules/mesh-build-go/buildstep/isolate.go index 2efa739ed..13deb72b2 100644 --- a/modules/mesh-build-go/buildstep/isolate.go +++ b/modules/mesh-build-go/buildstep/isolate.go @@ -27,16 +27,19 @@ const Bubblewrap = "bwrap" // - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries, // certificates, time zone), with a hosts file naming only the loopback; // - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step -// downloaded) read-only; +// downloaded) read-only, and the ReadOnlyInside ones (inside the Workspace) read-only over it; // - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone // (WriteCaches), so a candidate's code never changes what a later build reads. type Isolation struct { - Workspace string - Writable []string - ReadOnly []string - Caches []string - WriteCaches bool - Env []string + Workspace string + Writable []string + ReadOnly []string + // ReadOnlyInside are directories inside the Workspace or a Writable one bound read-only after them, so they stay + // read-only (a later bind of a path wins; a ReadOnly one inside the Workspace would be covered by it). + ReadOnlyInside []string + Caches []string + WriteCaches bool + Env []string // Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback. Hosts string } @@ -91,7 +94,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin for _, group := range []struct { what string dirs []string - }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} { + }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"read-only directory", iso.ReadOnlyInside}, {"cache", iso.Caches}} { for _, d := range group.dirs { if err := checkDir(group.what, d); err != nil { return nil, err @@ -118,6 +121,9 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin for _, d := range iso.Writable { argv = append(argv, "--bind", d, d) } + for _, d := range iso.ReadOnlyInside { + argv = append(argv, "--ro-bind", d, d) + } if iso.Hosts != "" { argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts") } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go index 615f8b36a..92dc78d3b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go @@ -31,7 +31,7 @@ func toolOf(t *testing.T, m *Module, name string) func(map[string]any) (any, err } // Seams review must 2 (issue 1921): the kept composers and programs are listed with their size and last use, dropped one -// by one, and swept when unused past KeptBound; a use keeps a program. +// by one, and swept with the workspaces when unused past KeptBound; a use keeps a program. func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) { m, _ := fetchModule(t, &fakeStep{}) now := time.Now().UTC().Truncate(time.Second) @@ -62,9 +62,8 @@ func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) { t.Error("the dropped program is still kept") } - if swept := m.SweepKept(now, KeptBound); len(swept) != 1 || swept[0].Key != "old" { - t.Fatalf("swept %+v, want the old composer only", swept) - } + // The sweep runs with the workspaces' (CleanWorkspaces, on its timer). + m.CleanWorkspaces(now) if got := m.Kept(); len(got) != 1 || got[0].Key != "recent" { t.Fatalf("kept after the sweep = %+v", got) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 03a21b9af..b6189674b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "os/user" @@ -58,7 +59,7 @@ func main() { m.Settings = m.Home + "/settings.json" m.ReadKept = readOf(stdio.State(KeptState)).Get m.ReadManifest = readOf(stdio.State(ManifestsState)).Get - m.ReadInput = readOf(stdio.State(InputsState)).Get + m.ReadInput = func(state, key string) (json.RawMessage, error) { return readOf(stdio.State(state)).Get(key) } m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap // Outside a launch by the runtime — a build, a check — there is no node and no state to write. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 6cf40a48e..b8dc764ac 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -93,8 +93,8 @@ type Module struct { // ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request // names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test). ReadManifest func(key string) (json.RawMessage, error) - // ReadInput reads a key of the checker's inputs state (InputsState), which a run ask's input names by its digest. - ReadInput func(key string) (json.RawMessage, error) + // ReadInput reads a key of a state the module reads (module.json), where a run ask's input names its digest. + ReadInput func(state, key string) (json.RawMessage, error) // PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test). PlaceHead func(ctx context.Context, h delivery.RunHead, dir string) error GoModCache string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index ce14ea74a..358dac35c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -4,7 +4,7 @@ package main // checking program of a group's check. The module builds the program at the commit the ask names, once per // delivery.RunKey, as a compose ask builds a composer (programOf); fetches every head the ask names and lays each read-only // under the directory MESH_RUN_HEADS names, at `/`; reads the input the ask names by its digest from the -// checker's inputs state and writes it to the file MESH_RUN_INPUT names; and runs the program shut off with the ask's +// state the ask names and writes it to the file MESH_RUN_INPUT names; and runs the program shut off with the ask's // arguments: no network, none of the account. The answer is delivery.RunAnswer: the program's exit, the last line of its // standard output and the first line of its standard error. A program that fails is an answer, not a fault of the step. @@ -22,15 +22,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// InputsState is the checker's state that keeps a run's input by its digest (the module reads it, module.json reads). -const InputsState = delivery.CheckerModule + ".inputs" - // run runs a run ask: the program of its commit, built once, over its heads and its input. func (m *Module) run(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) { job := *a.Run - if job.Input.State != InputsState { - return nil, buildstep.Refuse("the run ask %s keeps its input in %s; this module reads %s", a.ID, job.Input.State, InputsState) - } body, err := m.inputOf(job.Input) if err != nil { return nil, err @@ -53,17 +47,18 @@ func (m *Module) run(ctx context.Context, fetch buildstep.StepFunc, a delivery.S return ans, nil } -// inputOf is the input a run names, read from the checker's inputs state and checked against its digest and size. +// inputOf is the input a run names, read from the state the ask names (module.json reads it) and checked against its +// digest and size. func (m *Module) inputOf(in delivery.RunInput) ([]byte, error) { if m.ReadInput == nil { return nil, errors.New("this module reads no inputs state") } - raw, err := m.ReadInput(in.Digest) + raw, err := m.ReadInput(in.State, in.Digest) if err != nil { return nil, err } if raw == nil { - return nil, fmt.Errorf("the checker keeps no input of digest %s", in.Digest) + return nil, fmt.Errorf("%s keeps no input of digest %s", in.State, in.Digest) } if err := in.Verify(raw); err != nil { return nil, buildstep.Refuse("%v", err) @@ -115,10 +110,9 @@ func (m *Module) runProgram(ctx context.Context, exe string, job delivery.RunJob if !m.Inside { iso := m.isolation(id, false) iso.ReadOnly = append(iso.ReadOnly, exe) + iso.ReadOnlyInside = []string{heads} // bound after the workspace, so the program cannot change a head run = buildstep.Isolated(m.Run, iso) } - // The heads are bound read-only inside the writable workspace by running the program with them as its read-only - // directory too (a later bind of the same path wins). r := run(ctx, []string{delivery.RunHeadsEnv + "=" + heads, delivery.RunInputEnv + "=" + file}, exe, job.Args...) if r.Err != "" && r.Status == 0 { return delivery.RunAnswer{}, fmt.Errorf("the checking program could not run: %s", r.Err) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go index c1463d2a8..77775eb1e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go @@ -22,8 +22,8 @@ import ( // 1921/C3: a run ask put on the Go build seat's check stream is pulled by the module started whole on the real bus and // answered on its step key .: the program, run shut off over its heads and its input (read from the -// checker's inputs state by digest), answers its last line and exit; a program that dials the machine's network is -// shut off, and the answer says it failed. +// state the ask names by digest), answers its last line and exit; a program cannot write under its heads, and one +// that dials the machine's network is shut off, and the answer says it failed. func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { needTools(t, "the shut-off run", "bash", "bwrap") nc, err := nats.Connect(testbus.URL(t)) @@ -59,13 +59,13 @@ func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { } m.PlaceHead = func(_ context.Context, h delivery.RunHead, dir string) error { return os.MkdirAll(dir, 0o700) } body := `{"v":1}` - m.ReadInput = func(string) (json.RawMessage, error) { return json.RawMessage(body), nil } + m.ReadInput = func(string, string) (json.RawMessage, error) { return json.RawMessage(body), nil } a := runAskOf(t, body) key := delivery.RunKey(a.Run.Program, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}, nil) exe := m.programPath("programs", key) os.MkdirAll(filepath.Dir(exe), 0o700) - script := "#!/bin/bash\n" + `echo "start"; exec 3<>/dev/tcp/127.0.0.1/` + strings.Split(tcp.Addr().String(), ":")[1] + " || exit 7\n" + `echo '{"dialed":true}'` + "\n" + script := "#!/bin/bash\n" + `if touch "$MESH_RUN_HEADS/novox/mesh-sdk/x" 2>/dev/null; then echo '{"wrote":true}'; exit 9; fi; echo "start"; exec 3<>/dev/tcp/127.0.0.1/` + strings.Split(tcp.Addr().String(), ":")[1] + " || exit 7\n" + `echo '{"dialed":true}'` + "\n" if err := os.WriteFile(exe, []byte(script), 0o700); err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go index 10baa06a8..71c4a95eb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go @@ -12,13 +12,17 @@ import ( sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/composer" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" ) const runCommit = "5555555555555555555555555555555555555555" +// runInputs is the state a run ask's input is kept in, as the checker names it in the ask. +const runInputs = "mesh-delivery-checker.inputs" + func runAskOf(t *testing.T, body string) delivery.StepAsk { t.Helper() - in, err := delivery.RunInputOf(InputsState, []byte(body)) + in, err := delivery.RunInputOf(runInputs, []byte(body)) if err != nil { t.Fatal(err) } @@ -35,8 +39,8 @@ func runAskOf(t *testing.T, body string) delivery.StepAsk { func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { m := composeModule(t) body := `{"v":1,"case":"x"}` - m.ReadInput = func(key string) (json.RawMessage, error) { - if key != runAskOf(t, body).Run.Input.Digest { + m.ReadInput = func(state, key string) (json.RawMessage, error) { + if state != runInputs || key != runAskOf(t, body).Run.Input.Digest { return nil, nil } return json.RawMessage(body), nil @@ -68,15 +72,35 @@ func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { t.Errorf("the run's workspace was left: %v", entries) } - // An input other than the one named is refused, naming it; an unknown state is refused. - m.ReadInput = func(string) (json.RawMessage, error) { return json.RawMessage(`{"other":1}`), nil } + // Shut off, the heads are bound read-only after the writable workspace, so a write under MESH_RUN_HEADS fails: a + // read-only bind before the workspace's is covered by it. + m.Inside = false + var argv []string + m.Run = func(_ context.Context, _ []string, _ string, args ...string) forgekey.Ran { + argv = args + return forgekey.Ran{Status: 0, Stdout: "{}"} + } + if _, err := m.run(context.Background(), fetch, a); err != nil { + t.Fatal(err) + } + line, ws, heads := " "+strings.Join(argv, " ")+" ", -1, -1 + for i := 0; i+2 < len(argv); i++ { + switch { + case argv[i] == "--bind" && filepath.Dir(argv[i+1]) == m.Work: + ws = i + case argv[i] == "--ro-bind" && filepath.Base(argv[i+1]) == "heads" && argv[i+2] == argv[i+1]: + heads = i + } + } + if ws < 0 || heads < ws || !strings.Contains(line, " "+delivery.RunHeadsEnv+" "+argv[heads+1]+" ") { + t.Errorf("the heads are not bound read-only over the workspace: %s", line) + } + m.Inside, m.Run = true, StepRunner(t.TempDir()) + // An input other than the one named is refused, naming it. + m.ReadInput = func(string, string) (json.RawMessage, error) { return json.RawMessage(`{"other":1}`), nil } if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "not the one the ask names") { t.Errorf("a wrong input: %v", err) } - a.Run.Input.State = "mesh-delivery-indexer.manifests" - if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "this module reads") { - t.Errorf("another state: %v", err) - } } // 1921/C2 (F2): a compose ask's seats reach the composer: the module decodes and fills the request and writes it again, From 608c4d4be6c417e5997b6bec7a9dea779af1a40d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 23:26:12 +0100 Subject: [PATCH 07/10] Fail a run step whose answer is over the bus's value limit, naming its size (issue 1921, review round 1) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-go/cmd/mesh-build-go/run.go | 7 +++++++ modules/mesh-build-go/cmd/mesh-build-go/run_test.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index 358dac35c..375394b67 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -44,6 +44,13 @@ func (m *Module) run(ctx context.Context, fetch buildstep.StepFunc, a delivery.S return nil, err } ans.Built, ans.Key = built, key + // The answer is written on the step key as one value: one over the bus's value limit (the input's bound) fails the + // step, naming its size, before it is written. + if raw, err := json.Marshal(ans); err != nil { + return nil, err + } else if len(raw) > delivery.RunInputBound { + return nil, fmt.Errorf("the checking program's answer is %d bytes, over the bound of %d bytes a step key holds", len(raw), delivery.RunInputBound) + } return ans, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go index 71c4a95eb..961991794 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go @@ -95,6 +95,13 @@ func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { if ws < 0 || heads < ws || !strings.Contains(line, " "+delivery.RunHeadsEnv+" "+argv[heads+1]+" ") { t.Errorf("the heads are not bound read-only over the workspace: %s", line) } + // An answer over the bus's value limit fails the step, naming its size. + m.Run = func(context.Context, []string, string, ...string) forgekey.Ran { + return forgekey.Ran{Status: 0, Stdout: `"` + strings.Repeat("x", delivery.RunInputBound) + `"`} + } + if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "over the bound") { + t.Errorf("an answer over the bound: %v", err) + } m.Inside, m.Run = true, StepRunner(t.TempDir()) // An input other than the one named is refused, naming it. m.ReadInput = func(string, string) (json.RawMessage, error) { return json.RawMessage(`{"other":1}`), nil } From c8bcb2cdf5672239ab37e3b0cce1619403b83c04 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 6 Nov 2026 23:43:05 +0100 Subject: [PATCH 08/10] mesh-build-go asks a run's input of the seat verb the ask names, and reads no checker state (issue 1921, B0 review round 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator's answer: reading mesh-delivery-checker.inputs broke ADR 0596 §2. The manifest drops that read and invokes seat:mesh-delivery-checker.input; the run step asks the verb through the runtime (stdio.Ask) and takes only the bytes the ask names (delivery.RunInput.Read). Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-build-go/compose_test.go | 10 ++++- .../mesh-build-go/cmd/mesh-build-go/main.go | 3 +- .../mesh-build-go/cmd/mesh-build-go/module.go | 5 ++- .../mesh-build-go/cmd/mesh-build-go/run.go | 25 +++++------ .../cmd/mesh-build-go/run_integration_test.go | 42 +++++++++++++++++-- .../cmd/mesh-build-go/run_test.go | 41 ++++++++++++------ modules/mesh-build-go/module.json | 4 +- 7 files changed, 93 insertions(+), 37 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go index 61cdf6f30..6520a50ad 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go @@ -150,11 +150,17 @@ func TestAComposeAskFailsLoudly(t *testing.T) { } } -// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest. +// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest. 1921 (B0): a run's input +// is asked of the checker seat's verb input, as the manifest invokes it, and no state of the checker's inputs is read +// (ADR 0596 §2). func TestTheModuleReadsTheIndexersManifests(t *testing.T) { - if m := readManifest(t); !slices.Contains(m.Reads, ManifestsState) { + m := readManifest(t) + if !slices.Contains(m.Reads, ManifestsState) { t.Errorf("reads = %v, lacks %s", m.Reads, ManifestsState) } + if !slices.Contains(m.Invokes, runInputVerb) || slices.Contains(m.Reads, "mesh-delivery-checker.inputs") { + t.Errorf("invokes = %v, reads = %v: want %s invoked and no inputs state read", m.Invokes, m.Reads, runInputVerb) + } } // ADR 0593 §7 (C9): a compose ask naming the node-engine's repository builds its composer command in the Go module at the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index b6189674b..906b8081c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -7,7 +7,6 @@ package main import ( "context" - "encoding/json" "fmt" "os" "os/user" @@ -59,7 +58,7 @@ func main() { m.Settings = m.Home + "/settings.json" m.ReadKept = readOf(stdio.State(KeptState)).Get m.ReadManifest = readOf(stdio.State(ManifestsState)).Get - m.ReadInput = func(state, key string) (json.RawMessage, error) { return readOf(stdio.State(state)).Get(key) } + m.AskInput = stdio.Ask m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap // Outside a launch by the runtime — a build, a check — there is no node and no state to write. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index b8dc764ac..d3ad804ea 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -93,8 +93,9 @@ type Module struct { // ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request // names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test). ReadManifest func(key string) (json.RawMessage, error) - // ReadInput reads a key of a state the module reads (module.json), where a run ask's input names its digest. - ReadInput func(state, key string) (json.RawMessage, error) + // AskInput asks a seat verb the module invokes (module.json) as the runtime calls it (stdio.Ask): the verb a run ask's + // input names, answering the input by its digest (delivery.RunInput.Read). + AskInput func(key string, body any) (json.RawMessage, error) // PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test). PlaceHead func(ctx context.Context, h delivery.RunHead, dir string) error GoModCache string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index 375394b67..13270f6bd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -3,8 +3,9 @@ package main // **A run ask runs a program built from a change, over heads it reads** (novox/hq ADR 0644, ADR 0656; issue 1921): the // checking program of a group's check. The module builds the program at the commit the ask names, once per // delivery.RunKey, as a compose ask builds a composer (programOf); fetches every head the ask names and lays each read-only -// under the directory MESH_RUN_HEADS names, at `/`; reads the input the ask names by its digest from the -// state the ask names and writes it to the file MESH_RUN_INPUT names; and runs the program shut off with the ask's +// under the directory MESH_RUN_HEADS names, at `/`; asks the seat verb the ask's input names for it by its +// digest (as module.json invokes it; never a read of the asker's state, ADR 0596 §2) and writes it to the file +// MESH_RUN_INPUT names; and runs the program shut off with the ask's // arguments: no network, none of the account. The answer is delivery.RunAnswer: the program's exit, the last line of its // standard output and the first line of its standard error. A program that fails is an answer, not a fault of the step. @@ -54,23 +55,17 @@ func (m *Module) run(ctx context.Context, fetch buildstep.StepFunc, a delivery.S return ans, nil } -// inputOf is the input a run names, read from the state the ask names (module.json reads it) and checked against its -// digest and size. +// inputOf is the input a run names, asked of the seat verb the ask names (module.json invokes it) and checked against +// its digest and size: bytes that are not the input named refuse the ask. func (m *Module) inputOf(in delivery.RunInput) ([]byte, error) { - if m.ReadInput == nil { - return nil, errors.New("this module reads no inputs state") + if m.AskInput == nil { + return nil, errors.New("this module asks no seat verb for a run's input") } - raw, err := m.ReadInput(in.State, in.Digest) - if err != nil { - return nil, err - } - if raw == nil { - return nil, fmt.Errorf("%s keeps no input of digest %s", in.State, in.Digest) - } - if err := in.Verify(raw); err != nil { + raw, err := in.Read(m.AskInput) + if err != nil && raw != nil { return nil, buildstep.Refuse("%v", err) } - return raw, nil + return raw, err } // placeHead lays one head's commit as a plain directory at dir, with no symlink in it; a test sets its own. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go index 77775eb1e..14d4f8fbf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_integration_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "errors" "net" "os" "path/filepath" @@ -21,8 +22,9 @@ import ( ) // 1921/C3: a run ask put on the Go build seat's check stream is pulled by the module started whole on the real bus and -// answered on its step key .: the program, run shut off over its heads and its input (read from the -// state the ask names by digest), answers its last line and exit; a program cannot write under its heads, and one +// answered on its step key .: the program, run shut off over its heads and its input (asked by digest of the +// seat verb the ask names, which a fake checker answers on the bus as the checker's verb input does), answers its last +// line and exit; a program cannot write under its heads, and one // that dials the machine's network is shut off, and the answer says it failed. func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { needTools(t, "the shut-off run", "bash", "bwrap") @@ -59,7 +61,33 @@ func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { } m.PlaceHead = func(_ context.Context, h delivery.RunHead, dir string) error { return os.MkdirAll(dir, 0o700) } body := `{"v":1}` - m.ReadInput = func(string, string) (json.RawMessage, error) { return json.RawMessage(body), nil } + // The checker's verb input on the bus, and the module asking it as the runtime calls a seat verb it invokes. + asked := make(chan string, 4) + sub, err := nc.Subscribe("mesh.seat.mesh-delivery-checker.tool.input", func(msg *nats.Msg) { + var ask delivery.RunInputAsk + _ = json.Unmarshal(msg.Data, &ask) + asked <- ask.Digest + reply, _ := json.Marshal(map[string]any{"result": delivery.RunInputAnswer{Digest: ask.Digest, Input: []byte(body)}}) + _ = msg.Respond(reply) + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + m.AskInput = func(key string, b any) (json.RawMessage, error) { + if key != runInputVerb { + return nil, errors.New("asked " + key) + } + data, _ := json.Marshal(b) + msg, err := nc.Request("mesh.seat.mesh-delivery-checker.tool.input", data, 5*time.Second) + if err != nil { + return nil, err + } + var r struct { + Result json.RawMessage `json:"result"` + } + return r.Result, json.Unmarshal(msg.Data, &r) + } a := runAskOf(t, body) key := delivery.RunKey(a.Run.Program, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}, nil) @@ -102,4 +130,12 @@ func TestARunAskIsAnsweredOnTheBusAndItsProgramIsShutOff(t *testing.T) { if ans.Key != key || ans.Exit != 7 || strings.Contains(string(ans.Output), "dialed") || string(ans.Output) != `"start"` { t.Fatalf("the answer of a program that dials out = %+v output %s", ans, ans.Output) } + select { + case d := <-asked: + if d != a.Run.Input.Digest { + t.Errorf("the verb input was asked %s, want %s", d, a.Run.Input.Digest) + } + default: + t.Error("the run's input was not asked of the checker's verb input") + } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go index 961991794..31fd4c5a1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go @@ -2,13 +2,18 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" + "fmt" "os" "path/filepath" "strings" "testing" "time" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/composer" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -17,12 +22,23 @@ import ( const runCommit = "5555555555555555555555555555555555555555" -// runInputs is the state a run ask's input is kept in, as the checker names it in the ask. -const runInputs = "mesh-delivery-checker.inputs" +// runInputVerb is the seat verb a run ask's input is served on, as the checker names it in the ask. +const runInputVerb = "seat:mesh-delivery-checker.input" + +// answersInput is a seat verb serving body by its digest, as the checker's verb input answers (delivery.RunInputAnswer). +func answersInput(body string) func(key string, b any) (json.RawMessage, error) { + return func(key string, b any) (json.RawMessage, error) { + ask, _ := b.(delivery.RunInputAsk) + if sum := sha256.Sum256([]byte(body)); key != runInputVerb || ask.Digest != hex.EncodeToString(sum[:]) { + return nil, fmt.Errorf("the checker keeps no input %s", ask.Digest) + } + return json.Marshal(delivery.RunInputAnswer{Digest: ask.Digest, Input: []byte(body)}) + } +} func runAskOf(t *testing.T, body string) delivery.StepAsk { t.Helper() - in, err := delivery.RunInputOf(runInputs, []byte(body)) + in, err := delivery.RunInputOf(runInputVerb, []byte(body)) if err != nil { t.Fatal(err) } @@ -39,12 +55,7 @@ func runAskOf(t *testing.T, body string) delivery.StepAsk { func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { m := composeModule(t) body := `{"v":1,"case":"x"}` - m.ReadInput = func(state, key string) (json.RawMessage, error) { - if state != runInputs || key != runAskOf(t, body).Run.Input.Digest { - return nil, nil - } - return json.RawMessage(body), nil - } + m.AskInput = answersInput(body) m.PlaceHead = func(_ context.Context, h delivery.RunHead, dir string) error { return os.MkdirAll(dir, 0o700) } @@ -103,11 +114,17 @@ func TestAKeptProgramIsRunOverItsHeadsAndInput(t *testing.T) { t.Errorf("an answer over the bound: %v", err) } m.Inside, m.Run = true, StepRunner(t.TempDir()) - // An input other than the one named is refused, naming it. - m.ReadInput = func(string, string) (json.RawMessage, error) { return json.RawMessage(`{"other":1}`), nil } - if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "not the one the ask names") { + // An input other than the one named is refused, naming it; one the verb does not answer fails the step, saying why. + m.AskInput = func(_ string, b any) (json.RawMessage, error) { + return json.Marshal(delivery.RunInputAnswer{Digest: b.(delivery.RunInputAsk).Digest, Input: []byte(`{"other":1}`)}) + } + if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "not the one the ask names") || !errors.As(err, new(*buildstep.Refusal)) { t.Errorf("a wrong input: %v", err) } + m.AskInput = answersInput(`{"v":2}`) + if _, err := m.run(context.Background(), fetch, a); err == nil || !strings.Contains(err.Error(), "keeps no input") { + t.Errorf("an input the verb does not answer: %v", err) + } } // 1921/C2 (F2): a compose ask's seats reach the composer: the module decodes and fills the request and writes it again, diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 3281af8d3..3e30a5cb1 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -20,10 +20,12 @@ } ], "reads": [ - "mesh-delivery-checker.inputs", "mesh-delivery-checker.kept", "mesh-delivery-indexer.manifests" ], + "invokes": [ + "seat:mesh-delivery-checker.input" + ], "config": { "capacity": { "type": "number", From be7d649d1f0cb2e85eb77c4955a1de406755181c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 7 Nov 2026 00:21:17 +0100 Subject: [PATCH 09/10] Progress reads only a design's head, and says so beside the tracker's tree when even that fails (issue 1947) --- .../cmd/mesh-knowledge/design.go | 21 +++++++++++-- .../mesh-knowledge/cmd/mesh-knowledge/hq.go | 30 +++++++++++++++++-- .../cmd/mesh-knowledge/progress.go | 6 ++++ .../cmd/mesh-knowledge/progress_test.go | 19 ++++++++++++ modules/records/cmd/records/main.go | 11 +++++-- modules/records/cmd/records/records.go | 18 +++++++++-- modules/records/cmd/records/records_test.go | 4 +++ 7 files changed, 99 insertions(+), 10 deletions(-) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/design.go b/modules/mesh-knowledge/cmd/mesh-knowledge/design.go index 3c9d423b1..ff65eb194 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/design.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/design.go @@ -24,8 +24,15 @@ type Design struct { Status string Decisions []string Milestones []Milestone + // Unread says why the design's frontmatter and title could not be read, when they could not (novox/hq issue + // 1947); the progress answer carries it as a warning beside a tree built from the tracker's milestone records. + Unread string } +// DesignHeadBytes is how much of a design is read: its frontmatter and its title lead it, and its milestones are the +// tracker's records, so the rest is never needed. +const DesignHeadBytes = 16 << 10 + // Milestone is one milestone of a design, as the issue tracker keeps it. type Milestone struct { ID string @@ -151,11 +158,19 @@ func (k *KB) designs(all bool, number int) ([]*Design, string, error) { } d := c.byPath[p] if d == nil { - content, _, err := k.Records.Read(p) + var content string + var err error + if h, ok := k.Records.(HeadReader); ok { + content, _, err = h.ReadHead(p, DesignHeadBytes) + } else { + content, _, err = k.Records.Read(p) + } if err != nil { - return nil, "", fmt.Errorf("the records module does not read %s: %v", p, err) + d = parseDesign(p, "") + d.Unread = fmt.Sprintf("the records module does not read %s (%v): its title and state are not known here", p, err) + } else { + d = parseDesign(p, content) } - d = parseDesign(p, content) c.byPath[p] = d k.progReads++ } diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/hq.go b/modules/mesh-knowledge/cmd/mesh-knowledge/hq.go index fe0d8916b..d75927786 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/hq.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/hq.go @@ -22,6 +22,12 @@ type Records interface { Read(path string) (content, commit string, err error) } +// HeadReader is Records that can read only the first part of a document: a caller that needs a frontmatter or a +// title reads a long document through it, bounded, never whole (novox/hq issue 1947). +type HeadReader interface { + ReadHead(path string, bytes int) (content, commit string, err error) +} + // busRecords is Records over the bus: the records module's tools, called as this module. type busRecords struct{} @@ -58,8 +64,18 @@ func (busRecords) List(folder string) ([]string, []string, error) { return f.Folders, f.Documents, nil } -func (busRecords) Read(path string) (string, string, error) { - raw, err := stdio.Ask("records.records_read", map[string]any{"path": path}) +func (busRecords) Read(path string) (string, string, error) { return busRead(path, 0) } + +func (busRecords) ReadHead(path string, bytes int) (string, string, error) { + return busRead(path, bytes) +} + +func busRead(path string, head int) (string, string, error) { + args := map[string]any{"path": path} + if head > 0 { + args["head"] = head + } + raw, err := stdio.Ask("records.records_read", args) if err != nil { return "", "", err } @@ -118,11 +134,19 @@ func (d dirRecords) List(folder string) ([]string, []string, error) { return folders, docs, nil } -func (d dirRecords) Read(path string) (string, string, error) { +func (d dirRecords) Read(path string) (string, string, error) { return d.ReadHead(path, 0) } + +func (d dirRecords) ReadHead(path string, head int) (string, string, error) { raw, err := os.ReadFile(filepath.Join(d.dir, filepath.FromSlash(path))) if err != nil { return "", "", err } + if head > 0 && head < len(raw) { + raw = raw[:head] + if i := strings.LastIndexByte(string(raw), '\n'); i > 0 { + raw = raw[:i+1] + } + } commit, _ := d.Standing() return string(raw), commit, nil } diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go index ae26f2c30..bbb200b23 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go @@ -329,6 +329,12 @@ func (k *KB) progressDesign(p *Progress, number int, milestone string) error { return err } d := ds[0] + if d.Unread != "" { + p.Warnings = append(p.Warnings, d.Unread) + if d.Title == "" { + d.Title = fmt.Sprintf("design %d", d.Number) + } + } if len(d.Milestones) == 0 { return fmt.Errorf("design %d has no milestone records in the issue tracker: seed them from its table with "+ "issue-tracker.import milestones %d", number, number) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go index 1fdb0dc3a..5446d88ca 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go @@ -142,6 +142,19 @@ func (c *countingRecords) Read(p string) (string, string, error) { return content, c.commit, err } +func (c *countingRecords) ReadHead(p string, n int) (string, string, error) { + c.reads++ + content, _, err := c.dirRecords.ReadHead(p, n) + return content, c.commit, err +} + +// brokenRead is Records whose reads fail as the records module's do for a document longer than it answers whole. +type brokenRead struct{ dirRecords } + +func (brokenRead) ReadHead(p string, n int) (string, string, error) { + return "", "", fmt.Errorf("%s is longer than the records module answers whole", p) +} + // 1947/AC2, as issue 1781 changed it: a design's title, state and decisions are hq's; its milestones are the issue // tracker's records, ordered as its table, an id found with spaces or hyphens, and a closed one holding no issue done. func TestADesignsMilestonesAreTheTrackersRecords(t *testing.T) { @@ -316,6 +329,12 @@ func TestProgressOfADesignMinimalAndDetailed(t *testing.T) { if big, _ := k.Progress(ProgressAsk{Of: "design 47", Bound: 8192}); big.Cut != "" || big.Tree != p.Tree { t.Errorf("a minimal answer at bound 8192 is cut: %q", big.Cut) } + // A design whose head the records module cannot read is said in a warning beside the tracker-built tree, never a failure. + kb := &KB{Records: brokenRead{progressRecords()}, Seats: newSeatsWorld(t).ask, Now: k.Now} + if bp, err := kb.Progress(ProgressAsk{Of: "design 47"}); err != nil || !strings.Contains(strings.Join(bp.Warnings, "\n"), "does not read") || + !strings.Contains(bp.Tree, "design 47") || !strings.Contains(bp.Tree, "R6") { + t.Fatalf("an unreadable design: %v %+v", err, bp) + } lines := strings.Split(p.Tree, "\n") if !strings.HasPrefix(lines[0], "✗ design 47 · Delivery: from a commit to delivered — ") { t.Fatalf("first line %q", lines[0]) diff --git a/modules/records/cmd/records/main.go b/modules/records/cmd/records/main.go index 9dbe6c083..9be7e5161 100644 --- a/modules/records/cmd/records/main.go +++ b/modules/records/cmd/records/main.go @@ -53,8 +53,15 @@ func tools(r *Records) []stdio.Tool { }}, {Name: "records_read", Description: "One document, whole, by its path in the repository — a decision record, a design document, an issue report.", - Input: map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md")}, - Run: func(a map[string]any) (any, error) { return r.Read(strArg(a, "path")) }}, + Input: map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md"), + "head": map[string]any{"type": "number", "description": "only the first this many bytes, cut at a line end (optional); the answer says partial"}}, + Run: func(a map[string]any) (any, error) { + head := 0 + if v, ok := a["head"].(float64); ok { + head = int(v) + } + return r.ReadHead(strArg(a, "path"), head) + }}, {Name: "records_list", Description: "What a folder of the repository holds: its sub-folders and its documents. The root when no folder is named.", Input: map[string]any{"folder": str("a folder inside the repository (optional)")}, diff --git a/modules/records/cmd/records/records.go b/modules/records/cmd/records/records.go index 99b9d952d..defd9e9e2 100644 --- a/modules/records/cmd/records/records.go +++ b/modules/records/cmd/records/records.go @@ -322,12 +322,18 @@ type Document struct { Path string `json:"path"` Content string `json:"content"` Truncated bool `json:"truncated"` - Commit string `json:"commit"` + // Partial is true when only the first part the caller asked for (head) is answered; Truncated stays the answer's own cap. + Partial bool `json:"partial,omitempty"` + Commit string `json:"commit"` } // Read is one document, whole, or its first part with a note when it is very long. The path stays inside // the checkout: `..` and absolute paths are refused, not resolved. -func (r *Records) Read(p string) (Document, error) { +func (r *Records) Read(p string) (Document, error) { return r.ReadHead(p, 0) } + +// ReadHead is Read, or with head above zero only the first head bytes of the document, cut at a line end, so a +// caller that needs a frontmatter or a title reads a long document without its whole length (novox/hq issue 1947). +func (r *Records) ReadHead(p string, head int) (Document, error) { clean, ok := inside(p) if !ok || clean == "" { return Document{}, fmt.Errorf("%q is not a path inside the repository", p) @@ -337,6 +343,14 @@ func (r *Records) Read(p string) (Document, error) { return Document{}, fmt.Errorf("the repository holds no %s — `records_list` says what it holds", clean) } d := Document{Path: clean, Content: string(raw), Commit: r.Standing().Commit} + if head > 0 && head < len(d.Content) { + cut := d.Content[:head] + if i := strings.LastIndex(cut, "\n"); i > 0 { + cut = cut[:i+1] + } + d.Content, d.Partial = cut, true + return d, nil + } if len(d.Content) > MostBytes { d.Content = d.Content[:MostBytes] + "\n\n[… truncated; the document is longer than this answer carries]" d.Truncated = true diff --git a/modules/records/cmd/records/records_test.go b/modules/records/cmd/records/records_test.go index 87e0af803..2dab8135a 100644 --- a/modules/records/cmd/records/records_test.go +++ b/modules/records/cmd/records/records_test.go @@ -70,6 +70,10 @@ func TestAPhraseInOneDesignDocumentComesBackFromWhereItIsWritten(t *testing.T) { if err != nil || !strings.Contains(doc.Content, "We decided the thing") || doc.Truncated { t.Fatalf("%+v %v", doc, err) } + if head, err := r.ReadHead("02-DECISIONS/0001-a-decision.md", 30); err != nil || !head.Partial || head.Truncated || len(head.Content) > 30 || + !strings.HasSuffix(head.Content, "\n") || !strings.HasPrefix(doc.Content, head.Content) { + t.Fatalf("a head read: %+v %v", head, err) + } root, _ := r.List("") if strings.Join(root.Folders, ",") != "02-DECISIONS,03-DESIGN" || strings.Join(root.Documents, ",") != "README.md" { t.Fatalf("%+v", root) From 7c0ce2fe218ba860aa2fc90784951e91c03a7590 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 7 Nov 2026 00:25:02 +0100 Subject: [PATCH 10/10] Progress: an unread design is not cached and is warned of at every level; head is capped (issue 1947) --- .../cmd/mesh-knowledge/bus_test.go | 7 +++++-- .../cmd/mesh-knowledge/design.go | 21 +++++++++++++++++-- .../cmd/mesh-knowledge/progress.go | 2 ++ .../cmd/mesh-knowledge/progress_test.go | 15 +++++++++++++ modules/records/cmd/records/main.go | 2 +- modules/records/cmd/records/records.go | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/bus_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/bus_test.go index 03c87498e..9102337fb 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/bus_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/bus_test.go @@ -107,9 +107,12 @@ func newBusWorld(t *testing.T) *busWorld { return map[string]any{"folders": f, "documents": d}, err }) answer("records.records_read", func(body json.RawMessage) (any, error) { - var a struct{ Path string } + var a struct { + Path string + Head int + } _ = json.Unmarshal(body, &a) - c, commit, err := records.Read(a.Path) + c, commit, err := records.ReadHead(a.Path, a.Head) return map[string]any{"content": c, "commit": commit}, err }) j := &judge{} diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/design.go b/modules/mesh-knowledge/cmd/mesh-knowledge/design.go index ff65eb194..4f1159386 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/design.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/design.go @@ -29,6 +29,17 @@ type Design struct { Unread string } +// unreadWarnings says each design whose head was not read, so none drops out of an answer silently. +func unreadWarnings(ds []*Design) []string { + var out []string + for _, d := range ds { + if d.Unread != "" { + out = append(out, d.Unread) + } + } + return out +} + // DesignHeadBytes is how much of a design is read: its frontmatter and its title lead it, and its milestones are the // tracker's records, so the rest is never needed. const DesignHeadBytes = 16 << 10 @@ -58,12 +69,13 @@ func parseDesign(path, content string) *Design { d.Number, _ = strconv.Atoi(m[1]) } lines := strings.Split(content, "\n") - body := 0 + body, closed := 0, false if len(lines) > 0 && strings.TrimSpace(lines[0]) == "---" { for i := 1; i < len(lines); i++ { l := lines[i] if strings.TrimSpace(l) == "---" { body = i + 1 + closed = true break } if v, ok := strings.CutPrefix(l, "status:"); ok { @@ -74,6 +86,9 @@ func parseDesign(path, content string) *Design { } } } + if len(lines) > 0 && strings.TrimSpace(lines[0]) == "---" && !closed { + d.Unread = fmt.Sprintf("the frontmatter of %s does not close in the part read: its state and decisions are not known here", path) + } for i := body; i < len(lines) && d.Title == ""; i++ { if m := titleLine.FindStringSubmatch(strings.TrimSpace(lines[i])); m != nil { d.Title = strings.TrimSpace(m[1]) @@ -171,7 +186,9 @@ func (k *KB) designs(all bool, number int) ([]*Design, string, error) { } else { d = parseDesign(p, content) } - c.byPath[p] = d + if d.Unread == "" { + c.byPath[p] = d + } k.progReads++ } out = append(out, d) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go index bbb200b23..0f510275c 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go @@ -393,6 +393,7 @@ func (k *KB) progressMesh(p *Progress) error { return err } p.Source = fmt.Sprintf("the issue tracker's milestone records, and the designs in %s at hq %s", DesignsFolder, short(commit)) + p.Warnings = append(p.Warnings, unreadWarnings(ds)...) var with []*Design var without []string for _, d := range ds { @@ -520,6 +521,7 @@ func (k *KB) progressRecord(p *Progress, kind string, n int) error { return err } p.Source = fmt.Sprintf("the knowledge base, the designs at hq %s and the issue tracker's links", short(commit)) + p.Warnings = append(p.Warnings, unreadWarnings(ds)...) var designs []string for _, d := range ds { for _, dn := range d.Decisions { diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go index 5446d88ca..4511bc1ab 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go @@ -42,6 +42,15 @@ func (w withDesigns) Read(path string) (string, string, error) { return c, commit, err } +func (w withDesigns) ReadHead(path string, n int) (string, string, error) { + if c, commit, err := w.base.ReadHead(path, n); err == nil { + return c, commit, nil + } + c, _, err := w.designs.ReadHead(path, n) + commit, _ := w.base.Standing() + return c, commit, err +} + // seatsWorld plays the issue tracker and mesh-delivery from the captured answers, and counts what it is asked. type seatsWorld struct { mu sync.Mutex @@ -335,6 +344,12 @@ func TestProgressOfADesignMinimalAndDetailed(t *testing.T) { !strings.Contains(bp.Tree, "design 47") || !strings.Contains(bp.Tree, "R6") { t.Fatalf("an unreadable design: %v %+v", err, bp) } + if mp, err := kb.Progress(ProgressAsk{Of: "mesh"}); err != nil || !strings.Contains(strings.Join(mp.Warnings, "\n"), "does not read") { + t.Fatalf("the mesh level drops an unread design silently: %v %v", err, mp.Warnings) + } + if d := parseDesign(DesignsFolder+"/47-x.md", "---\nstatus: in-progress\n"); d.Unread == "" { + t.Errorf("a frontmatter cut before its close is not said") + } lines := strings.Split(p.Tree, "\n") if !strings.HasPrefix(lines[0], "✗ design 47 · Delivery: from a commit to delivered — ") { t.Fatalf("first line %q", lines[0]) diff --git a/modules/records/cmd/records/main.go b/modules/records/cmd/records/main.go index 9be7e5161..718120d66 100644 --- a/modules/records/cmd/records/main.go +++ b/modules/records/cmd/records/main.go @@ -52,7 +52,7 @@ func tools(r *Records) []stdio.Tool { return r.Search(strArg(a, "query"), limit) }}, {Name: "records_read", - Description: "One document, whole, by its path in the repository — a decision record, a design document, an issue report.", + Description: "One document by its path (whole, or its first part with head) in the repository — a decision record, a design document, an issue report.", Input: map[string]any{"path": str("the document's path, e.g. 02-DECISIONS/0025-....md"), "head": map[string]any{"type": "number", "description": "only the first this many bytes, cut at a line end (optional); the answer says partial"}}, Run: func(a map[string]any) (any, error) { diff --git a/modules/records/cmd/records/records.go b/modules/records/cmd/records/records.go index defd9e9e2..f18ed958d 100644 --- a/modules/records/cmd/records/records.go +++ b/modules/records/cmd/records/records.go @@ -343,6 +343,7 @@ func (r *Records) ReadHead(p string, head int) (Document, error) { return Document{}, fmt.Errorf("the repository holds no %s — `records_list` says what it holds", clean) } d := Document{Path: clean, Content: string(raw), Commit: r.Standing().Commit} + head = min(head, MostBytes) if head > 0 && head < len(d.Content) { cut := d.Content[:head] if i := strings.LastIndex(cut, "\n"); i > 0 {