BM4 (sdk): a run's test output is kept as <node>.<ask> and read by the verb output

A test step takes the level's text out of its test answers (their member
output) and keeps it in the holder's outputs objects as <node>.<ask id>, a
header of its size and the failing tests with their words from the level's
results beside the text; a check ask's answer keeps none. The verb output at
the part's form 2 takes the ask and answers the text's last 64 KB, its size and
the failing tests from the header; no-longer-kept only when this node's steps
key shows the ask ended more than the max-age (7 days) ago, an ask never run
here refused naming it.

Builds: design 47 H2 §791, §820; H-A-367 (1), H-A-412 (1) and its second and third addenda
This commit is contained in:
2026-11-22 01:12:46 +01:00
parent fad9f059ee
commit 02ec2f00d9
7 changed files with 296 additions and 3 deletions
@@ -156,6 +156,23 @@ type StepEnded struct {
At time.Time `json:"at,omitzero"`
}
// FailingTest is a failing test and its words, as a run's output names it (design 47 H2 §791).
type FailingTest struct {
Test string `json:"test"`
Words string `json:"words,omitempty"`
}
// OutputAnswer is what a build seat's verb output answers (design 47 H2 §791, H-A-367 (1), H-A-412): the ask, the run's
// test output's last 64 KB, its whole size and the failing tests with their words, or NoLongerKept once the output is past
// the outputs objects' max-age.
type OutputAnswer struct {
Ask string `json:"ask"`
Output string `json:"output,omitempty"`
Size int `json:"size,omitempty"`
Failing []FailingTest `json:"failing,omitempty"`
NoLongerKept bool `json:"no-longer-kept,omitempty"`
}
// NotRun is a test that did not run and why (design 47 H2 §800).
type NotRun struct {
Test string `json:"test"`
+35 -2
View File
@@ -60,6 +60,9 @@ type AskSteps struct {
// event step-ended on that node's subject (sdkstep.StepEndedSubject), under the ask's id. Nil runs no step ask.
Node string
SayStep func(end sdkstep.StepEnded) error
// Outputs are the holder's outputs objects (OutputsStore), where a test step keeps its run's output as `<node>.<ask
// id>` (design 47 H2 §791, H-A-412), which the seat's verb output reads (OutputTool). Nil keeps none, said in the journal.
Outputs OutputObjects
}
// A-489b (G-22-lead-answers-2.md): SaysItsEnds is nil when these steps can be asked a pipeline step and say its end: a node to name in the end and a SayStep to
@@ -319,6 +322,13 @@ func (s AskSteps) build(ctx context.Context, job sdkstep.BuildJob, version strin
}
func (s AskSteps) check(ctx context.Context, job sdkstep.CheckJob) (any, error) {
return s.checkTexts(ctx, job, nil)
}
// checkTexts is check, the text of each level's test answers (their member `output`) taken out of them and written to
// text, nil dropping it (design 47 H2 H-A-412 third addendum): a run's text is a test step's output alone, kept in the
// outputs objects, and a check ask's answer keeps none.
func (s AskSteps) checkTexts(ctx context.Context, job sdkstep.CheckJob, text *strings.Builder) (any, error) {
if _, err := s.onToolchain(ctx, job.Toolchain); err != nil {
return nil, err
}
@@ -345,7 +355,7 @@ func (s AskSteps) check(ctx context.Context, job sdkstep.CheckJob) (any, error)
// 2026-11-02: "A failed check shows only its first fault").
for _, level := range job.Levels {
la := sdkstep.LevelAnswer{Level: level}
_, err := s.level(ctx, job, level, f, runner, &la)
_, err := s.level(ctx, job, level, f, runner, &la, text)
answer.Levels = append(answer.Levels, la)
if err != nil {
// The fetch merged: its answer is kept on the failed step's end, so the checker previews the merge and judges its
@@ -357,7 +367,7 @@ func (s AskSteps) check(ctx context.Context, job sdkstep.CheckJob) (any, error)
}
// level runs one level of a check ask and says whether it passed.
func (s AskSteps) level(ctx context.Context, job sdkstep.CheckJob, level string, f fetched, runner string, la *sdkstep.LevelAnswer) (bool, error) {
func (s AskSteps) level(ctx context.Context, job sdkstep.CheckJob, level string, f fetched, runner string, la *sdkstep.LevelAnswer, text *strings.Builder) (bool, error) {
withRunner := func(args map[string]any) map[string]any {
if runner != "" {
args["runner"] = runner
@@ -386,6 +396,7 @@ func (s AskSteps) level(ctx context.Context, job sdkstep.CheckJob, level string,
if err != nil {
return false, err
}
r = takeText(r, text)
la.Tests = append(la.Tests, r)
var t struct {
Passed bool `json:"passed"`
@@ -612,3 +623,25 @@ func changesTestIn(changed []string, dir string) bool {
}
return false
}
// takeText is a test answer without its member `output`, the level's text, which it writes to text (nil drops it): design 47
// H2 H-A-412 third addendum. An answer with no text, or that is no object, is answered as it is.
func takeText(r json.RawMessage, text *strings.Builder) json.RawMessage {
var m map[string]json.RawMessage
if json.Unmarshal(r, &m) != nil {
return r
}
out, ok := m["output"]
if !ok {
return r
}
var t string
if json.Unmarshal(out, &t) == nil && text != nil {
text.WriteString(t)
}
delete(m, "output")
if stripped, err := json.Marshal(m); err == nil {
return stripped
}
return r
}
@@ -210,6 +210,17 @@ test {"level":"integration","packages":["x/pkg/a"],"workspace":"/w"}`
t.Fatalf("a level that errors: answer %+v, err %v; want the fetch and the unit level beside the error", out, err)
}
// Design 47 H2 H-A-412 third addendum: a check ask's answer keeps none of the level's text, its test answers' `output`.
steps = r.steps()
steps.Test = func(context.Context, map[string]any) (any, error) {
return map[string]any{"passed": true, "tests": 1, "output": "=== RUN TestA\n--- PASS: TestA\n"}, nil
}
out, err = RunAsk(context.Background(), steps, ask(sdkstep.CheckJob{Levels: []string{sdkstep.LevelUnit}, Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: runCommit}}))
if lv := out.(sdkstep.CheckAnswer).Levels; err != nil || len(lv) != 1 || len(lv[0].Tests) == 0 || strings.Contains(string(lv[0].Tests[0]), "output") ||
!strings.Contains(string(lv[0].Tests[0]), `"passed":true`) {
t.Fatalf("a check's answer %+v, %v; want its test answers without their text", out, err)
}
// ADR 0567 §12: a check's node names reach the digest and test tools, for the names check (a kept verdict misses when
// they change).
r.calls = nil
@@ -453,6 +464,8 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
archive string // the step's install archive, which every test call names
install StepFunc
arch string // the archive the end names
text string // the run's output kept as <node>.<ask id> (design 47 H2 §791), "" for none checked
failing []sdkstep.FailingTest
}{
{name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}),
digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}},
@@ -479,6 +492,11 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
saidOpen: "the bus did not answer in setup within 5m0s under load"},
{name: "a starved setup beside an assertion: failed", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 1, outcome: sdkstep.OutcomeFailed, counts: 2,
test: map[string]any{"passed": false, "tests": 2, "why": "2 failed", "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: words"}, {Package: "x/b", Test: "TestU", Cause: "got 1, want 2"}}}},
// §791, H-A-412: the level's text is kept as the run's output <node>.<ask id>, its header naming the failing tests.
{name: "failed with its text: the output kept with its failing tests", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, outcome: sdkstep.OutcomeFailed, counts: 1,
test: map[string]any{"passed": false, "tests": 1, "why": "1 failed", "output": "--- FAIL: TestU\n u_test.go:9: got 1, want 2\n",
"failed": []sdkstep.Failure{{Package: "x/b", Test: "TestU", Cause: "u_test.go:9: got 1, want 2"}}},
text: "--- FAIL: TestU\n u_test.go:9: got 1, want 2\n", failing: []sdkstep.FailingTest{{Test: "x/b TestU", Words: "u_test.go:9: got 1, want 2"}}},
// §800: the install archive the step names reaches the test calls, for the build module to unpack.
{name: "integration with an install archive: the test calls name it", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, tests: map[string]bool{"x/b": true},
outcome: sdkstep.OutcomePassed, counts: 1, kept: []string{"x/b"}, archive: "oci-artifact-registry://gitea/npm-tree:0.1.4"},
@@ -503,6 +521,8 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
s.Fetch = func(context.Context, map[string]any) (any, error) { return nil, row.fetchErr }
}
s.Lint, s.Node, s.Install = row.lint, "ace", row.install
outputs := fakeOutputs{m: map[string][]byte{}}
s.Outputs = outputs
if row.noTests {
s.Test = func(context.Context, map[string]any) (any, error) {
return map[string]any{"passed": true, "tests": 0, "no_tests": true}, nil
@@ -574,6 +594,12 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
if !slices.Equal(e.NotRun, row.notRun) || !strings.HasPrefix(e.Said, row.saidOpen) {
t.Errorf("%s: not run %+v said %q, want %+v and said opening %q", row.name, e.NotRun, e.Said, row.notRun, row.saidOpen)
}
if row.text != "" {
o, err := readOutput(outputs, "ace", row.ask.ID, nil)
if err != nil || o.Output != row.text || o.Size != len(row.text) || !slices.Equal(o.Failing, row.failing) {
t.Errorf("%s: the output kept answers %+v, %v; want %q with its size and failing %+v", row.name, o, err, row.text, row.failing)
}
}
}
}
@@ -1,8 +1,14 @@
package steprun
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"time"
"unicode/utf8"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
@@ -47,3 +53,120 @@ func ReadAnswerFrom(o OutputObjects) func(object string) ([]byte, error) {
return got.Data, nil
}
}
// OutputsMaxAge is the outputs objects' max-age (design 47 H2 §820: 7 days), past which the bus removes a run's output and
// the verb output answers it no longer kept.
const OutputsMaxAge = 7 * 24 * time.Hour
// OutputTail is how much of a run's output the verb output answers: its last 64 KB (design 47 H2 §791).
const OutputTail = 64 << 10
// outputHeader is the header a run's output object holds beside its text (design 47 H2 H-A-412 (1)): the text's size and the
// failing tests with their words taken from the level's results, so the verb output answers them without reading the text
// again. The object is the header's JSON on its first line, then the text.
type outputHeader struct {
Size int `json:"size"`
Failing []sdkstep.FailingTest `json:"failing,omitempty"`
}
// keepOutput writes a run's test output as the object `<node>.<ask id>` in the outputs objects (design 47 H2 §791,
// H-A-412): the header of its size and failing tests, then the text.
func keepOutput(o OutputObjects, node, ask, text string, failing []sdkstep.FailingTest) error {
if o == nil {
return fmt.Errorf("this module keeps no %s store", OutputsStore)
}
name, err := sdkstep.StepKey(node, ask)
if err != nil {
return err
}
head, err := json.Marshal(outputHeader{Size: len(text), Failing: failing})
if err != nil {
return err
}
_, err = o.Put(name, append(append(head, '\n'), text...), "a run's test output, named by its step key")
return err
}
// EndedHere is the verb output's read of this node's steps state (design 47 H2 H-A-412 second addendum (b)): whether a key
// of keys holds the ask's step on node, and when it ended (zero while it runs). keys and get read the holder's steps state.
func EndedHere(node string, keys func() ([]string, error), get func(key string) (json.RawMessage, error)) func(ask string) (time.Time, bool, error) {
return func(ask string) (time.Time, bool, error) {
if keys == nil || get == nil {
return time.Time{}, false, errors.New("this module reads no steps state")
}
all, err := keys()
if err != nil {
return time.Time{}, false, fmt.Errorf("the steps state's keys: %w", err)
}
for _, k := range all {
if _, n, id, perr := sdkstep.ParseStepKey(k); perr != nil || n != node || id != ask {
continue
}
raw, err := get(k)
if err != nil {
return time.Time{}, false, fmt.Errorf("the step key %s: %w", k, err)
}
if raw == nil {
continue
}
step, err := sdkstep.DecodeStep(raw)
if err != nil {
return time.Time{}, false, fmt.Errorf("the step key %s: %w", k, err)
}
if !sdkstep.Ended(step) {
return time.Time{}, true, nil
}
return step.Ended, true, nil
}
return time.Time{}, false, nil
}
}
// readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `<node>.<ask>`, the text's last
// OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key
// shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one
// whose output this node did not keep).
func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) {
if o == nil {
return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore)
}
name, err := sdkstep.StepKey(node, ask)
if err != nil {
return sdkstep.OutputAnswer{}, err
}
got, gerr := o.Get(name)
if gerr == nil {
return outputOf(ask, got.Data)
}
if ended == nil {
return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr)
}
at, here, err := ended(ask)
switch {
case err != nil:
return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s is not in the %s store (%v), and this node's steps state cannot say why: %w", name, OutputsStore, gerr, err)
case !here:
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s never ran on %s: no step key of it here", ask, node)
case !at.IsZero() && now().Sub(at) > OutputsMaxAge:
return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil
default:
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr)
}
}
// outputOf is the verb output's answer from an output object's bytes: its header and the text's last OutputTail bytes,
// cut at a whole character.
func outputOf(ask string, data []byte) (sdkstep.OutputAnswer, error) {
line, text, ok := bytes.Cut(data, []byte{'\n'})
var h outputHeader
if !ok || json.Unmarshal(line, &h) != nil {
return sdkstep.OutputAnswer{}, fmt.Errorf("the output of the ask %s holds no header", ask)
}
if len(text) > OutputTail {
text = text[len(text)-OutputTail:]
for len(text) > 0 && !utf8.RuneStart(text[0]) {
text = text[1:]
}
}
return sdkstep.OutputAnswer{Ask: ask, Output: string(text), Size: h.Size, Failing: h.Failing}, nil
}
@@ -1,10 +1,14 @@
package steprun
import (
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
@@ -41,4 +45,58 @@ func TestOutputsKeptAreRead(t *testing.T) {
if _, err := ReadAnswerFrom(nil)("o"); err == nil {
t.Fatal("no store read an answer")
}
// Design 47 H2 §791, H-A-412: the verb output answers a run's output `<node>.<ask>` from its header and its text's last
// 64 KB; with no object, no longer kept once this node's steps key shows the ask ended more than the max-age ago, and
// refused for an ask never run here or one whose output this node did not keep.
was := now
defer func() { now = was }()
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
now = func() time.Time { return at }
long := strings.Repeat("x", OutputTail) + "the end\n"
failing := []sdkstep.FailingTest{{Test: "x/b TestU", Words: "got 1, want 2"}}
out := fakeOutputs{m: map[string][]byte{}}
if err := keepOutput(out, "ace", "a1", long, failing); err != nil {
t.Fatal(err)
}
step := func(ended time.Time) json.RawMessage {
raw, _ := sdkstep.EncodeStep(sdkstep.Step{ID: "a2", Tool: "step", Node: "ace", State: sdkstep.StepPassed, Started: ended, Ended: ended})
return raw
}
steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)),
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))}
keys := func() ([]string, error) {
return slices.Collect(func(y func(string) bool) {
for k := range steps {
y(k)
}
}), nil
}
tool := OutputTool("node-build-go", "ace", out, EndedHere("ace", keys, func(k string) (json.RawMessage, error) { return steps[k], nil }))
if tool.Name != "node-build-go.output" {
t.Fatalf("name %q", tool.Name)
}
for _, c := range []struct {
ask, fail string
want sdkstep.OutputAnswer
}{
{ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}},
{ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}},
{ask: "a3", fail: "ran on ace, and its output is not in the outputs store"},
{ask: "a4", fail: "never ran on ace"},
{ask: "a5", fail: "never ran on ace"},
{fail: "asked for no ask"},
} {
got, err := tool.Run(map[string]any{"ask": c.ask})
if c.fail != "" {
if err == nil || !strings.Contains(err.Error(), c.fail) {
t.Errorf("output of %q: %+v, %v; want refused saying %q", c.ask, got, err, c.fail)
}
continue
}
o, _ := got.(sdkstep.OutputAnswer)
if err != nil || o.Ask != c.want.Ask || o.Output != c.want.Output || o.Size != c.want.Size || !slices.Equal(o.Failing, c.want.Failing) || o.NoLongerKept != c.want.NoLongerKept {
t.Errorf("output of %q: %+v, %v; want %+v", c.ask, got, err, c.want)
}
}
}
@@ -3,6 +3,7 @@ package steprun
import (
"encoding/base64"
"errors"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -85,3 +86,22 @@ func AnswerTool(seat string, read func(object string) ([]byte, error)) stdio.Too
return map[string]any{"ask": ask, "file": base64.StdEncoding.EncodeToString(body)}, nil
}}
}
// OutputTool serves `<seat>.output` at the part's form 2 (design 47 H2 §791, H-A-367 (1), H-A-412): a run's test output by
// its ask, from the object `<node>.<ask>` in the outputs objects o: its last 64 KB, its size and the failing tests with their
// words; no-longer-kept once ended (EndedHere over the holder's steps state) shows the ask ended on node more than the
// objects' max-age ago; an ask never run here is refused, naming it.
func OutputTool(seat, node string, o OutputObjects, ended func(ask string) (time.Time, bool, error)) stdio.Tool {
return stdio.Tool{Name: seat + "." + sdkstep.OutputVerb,
Description: "A run's test output by its ask: the output's last 64 KB, its size and the failing tests with their words, " +
"or no longer kept once past the outputs objects' max-age (7 days). An ask that never ran on this node is refused.",
Input: map[string]any{"type": "object", "required": []string{"ask"}, "properties": map[string]any{
"ask": map[string]any{"type": "string", "description": "the ask's id"}}},
Run: func(args map[string]any) (any, error) {
ask, _ := args["ask"].(string)
if ask == "" {
return nil, errors.New("the verb output is asked for no ask")
}
return readOutput(o, node, ask, ended)
}}
}
+17 -1
View File
@@ -165,7 +165,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
return 0, false, nil
}
job := sdkstep.CheckJob{Fetch: *withPackages(&run.Fetch, ask.Packages), Levels: []string{level}, Kept: true, Toolchain: run.Toolchain, Archive: run.Archive}
out, err := s.check(ctx, job)
// The level's text, taken out of its test answers, is the run's output (design 47 H2 H-A-412).
var text strings.Builder
out, err := s.checkTexts(ctx, job, &text)
if err != nil {
return err
}
@@ -217,6 +219,7 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
}
}
}
s.keepRunOutput(ask.ID, text.String(), failed)
if !passed && starvedOnly(failed) {
// Design 47 H2 §800, H-A-368: a setup whose wait on the bus passed its bound runs again by §799's delays, never
// failed; the third such run ends could-not-run (runStep).
@@ -255,6 +258,19 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
return nil
}
// keepRunOutput keeps a test step's level text as the run's output `<node>.<ask id>` in the outputs objects, with the
// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that
// keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results.
func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) {
var failing []sdkstep.FailingTest
for _, f := range failed {
failing = append(failing, sdkstep.FailingTest{Test: strings.TrimSpace(f.Package + " " + f.Test), Words: f.Cause})
}
if err := keepOutput(s.Outputs, s.Node, ask, text, failing); err != nil && s.Logf != nil {
s.Logf("the output of the ask %s could not be kept: %v", ask, err)
}
}
// errSetupStarved opens the why of a level whose every failure is a starved setup (design 47 H2 §800): its words are the
// could-not-run end's at the third run.
var errSetupStarved = errors.New("the bus did not answer in setup within")