diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go index e3f8ade3c..17f9d6110 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance.go @@ -6,7 +6,7 @@ // // A catalogue check (mesh-catalog checks/failurekit) refuses a build module with a test step whose tests do not call Run, // and any build module whose tests do not call LeftRunning, the kit's case of steps a previous process left running -// (leftrunning.go, issue 1890). +// (leftrunning.go, issue 1890). SeatVerbs (forms.go) is the case of the build-step part's form 2 (design 47 H2, H-A-421). package conformance import ( @@ -284,13 +284,26 @@ func judgePassing(c PassingCase, failed Failed) string { // A-489b: StepEnds is the case every build module runs on the steps it asks with: a pipeline step ask must be able to say its end // (steprun.AskSteps.SaysItsEnds), or the module would refuse every step ask the runner sends, as mesh-build-node did on its // first live pipeline (design 47 G §24). A catalogue check (failurekit) refuses a build module whose tests do not call it. +// +// H-A-421 (i): the steps must also keep a run's output (AskSteps.Outputs, design 47 H2 §791), or every run's output is lost. func StepEnds(t *testing.T, steps steprun.AskSteps) { t.Helper() - if err := steps.SaysItsEnds(); err != nil { - t.Errorf("the build module's steps cannot say a step's end: %v", err) + for _, p := range stepEnds(steps) { + t.Error(p) } } +func stepEnds(steps steprun.AskSteps) []string { + var problems []string + if err := steps.SaysItsEnds(); err != nil { + problems = append(problems, fmt.Sprintf("the build module's steps cannot say a step's end: %v", err)) + } + if steps.Outputs == nil { + problems = append(problems, "the build module's steps keep no outputs (AskSteps.Outputs): a run's output is never kept in its outputs objects") + } + return problems +} + // A-489c: RunsCounted is the case every build module runs on its puller (A-489a): the poison rule and a transport fault's run count the // runs the module's steps state says started, so the puller is built with Prior over the module's own read of that state. The // module builds its puller with read as its steps state's read; a step key it holds with a started run must count as that run. diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go index a3c80234b..d2c4c76a5 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go @@ -12,6 +12,8 @@ import ( buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" ) // The failure contract (issue 1901): the SDK's own parsers keep it on every captured case of the kit (1901/AC6); the @@ -204,7 +206,85 @@ func TestStepEndsRefusesStepsThatCannotSayTheirEnd(t *testing.T) { t.Errorf("%s: SaysItsEnds = %v, want refused %v", c.name, c.s.SaysItsEnds(), c.bad) } } - StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say}) + StepEnds(t, steprun.AskSteps{Node: "n", SayStep: say, Outputs: kitObjects{}}) + if p := stepEnds(steprun.AskSteps{Node: "n", SayStep: say}); len(p) != 1 || !strings.Contains(p[0], "keep no outputs") { + t.Errorf("steps with no outputs: %q, want refused for it (H-A-421 (i))", p) + } + + // H-A-421: SeatVerbs passes the SDK's own seat tools and steps wired to one outputs store, and refuses each form missing. + wired := func(seat string, o steprun.OutputObjects, install bool) ([]stdio.Tool, steprun.AskSteps) { + p := &steprun.Puller{Node: "kitnode", Capacity: func() int { return 1 }} + steps := steprun.AskSteps{Node: "kitnode", SayStep: say, Outputs: o} + if install { + steps.Install = func(context.Context, map[string]any) (any, error) { return nil, errors.New("the module's own install") } + } + ended := func(string) (time.Time, bool, error) { return time.Time{}, false, nil } + tools := append(steprun.SeatTools(seat, "kit", p), steprun.OutputTool(seat, "kitnode", o, ended), steprun.AnswerTool(seat, steprun.ReadAnswerFrom(o))) + return tools, steps + } + for _, c := range []struct { + name, seat, problem string + install bool + change func(tools []stdio.Tool, steps *steprun.AskSteps) []stdio.Tool + }{ + {name: "the Go seat, wired", seat: "node-build-go"}, + {name: "the Node seat with its install, wired", seat: "node-build-node", install: true}, + {name: "an install on the Go seat", seat: "node-build-go", install: true, problem: "only node-build-node accepts the kind install"}, + {name: "no outputs", seat: "node-build-go", problem: "keep no outputs", change: func(tools []stdio.Tool, steps *steprun.AskSteps) []stdio.Tool { + steps.Outputs = nil + return tools + }}, + {name: "steps and output on two stores", seat: "node-build-go", problem: "want its output kept", change: func(tools []stdio.Tool, steps *steprun.AskSteps) []stdio.Tool { + steps.Outputs = kitObjects{} + return tools + }}, + {name: "output of form 1", seat: "node-build-go", problem: "want the ask alone", change: func(tools []stdio.Tool, _ *steprun.AskSteps) []stdio.Tool { + for i := range tools { + if tools[i].Name == "node-build-go.output" { + tools[i].Input = map[string]any{"required": []string{"ask", "call"}, "properties": map[string]any{"ask": nil, "call": nil}} + } + } + return tools + }}, + {name: "no answer verb", seat: "node-build-go", problem: "serve no node-build-go.answer", change: func(tools []stdio.Tool, _ *steprun.AskSteps) []stdio.Tool { + return slices.DeleteFunc(tools, func(t stdio.Tool) bool { return t.Name == "node-build-go.answer" }) + }}, + {name: "current of form 1", seat: "node-build-go", problem: "answers no takes", change: func(tools []stdio.Tool, _ *steprun.AskSteps) []stdio.Tool { + for i := range tools { + if tools[i].Name == "node-build-go.current" { + tools[i].Run = func(map[string]any) (any, error) { return map[string]any{"node": "kitnode", "capacity": 1}, nil } + } + } + return tools + }}, + } { + tools, steps := wired(c.seat, kitObjects{}, c.install) + if c.change != nil { + tools = c.change(tools, &steps) + } + p := JudgeSeatVerbs(c.seat, tools, steps) + if c.problem == "" && len(p) > 0 || c.problem != "" && !slices.ContainsFunc(p, func(s string) bool { return strings.Contains(s, c.problem) }) { + t.Errorf("%s: problems %q, want %q", c.name, p, c.problem) + } + } + tools, steps := wired("node-build-image", kitObjects{}, false) + SeatVerbs(t, "node-build-image", tools, steps) +} + +// kitObjects is an outputs store in memory; each value is its own store. +type kitObjects map[string][]byte + +func (k kitObjects) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) { + k[name] = data + return buscalls.ObjectInfo{Name: name}, nil +} + +func (k kitObjects) Get(name string) (buscalls.ObjectData, error) { + d, ok := k[name] + if !ok { + return buscalls.ObjectData{}, errors.New("object not found") + } + return buscalls.ObjectData{Data: d}, nil } // A puller built without Prior, or with one that does not read the module's steps state, fails the case. diff --git a/modules/mesh-delivery-sdk/steprun/conformance/forms.go b/modules/mesh-delivery-sdk/steprun/conformance/forms.go new file mode 100644 index 000000000..81a09e079 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/conformance/forms.go @@ -0,0 +1,212 @@ +package conformance + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// SeatVerbs is the case every build module runs on its seat tools and the steps it asks with (design 47 H2 §791, §800, +// §801; H-A-421): it judges the build-step part's form 2 as the module serves it. current answers the node, whether it +// takes asks, since when, its capacity and the asks it runs, each with its asker; output takes the ask alone and answers a +// run's output, or refuses an ask never run here; answer takes the ask and the call of its chunks. A pipeline step is run +// through steps, their fetch and test bodies stood in by the kit's (a test answer holding the level's text in `output`, a +// NOT RUN skip and a failure, the step naming an install archive): its end must carry the not-run, the archive must reach +// the test call, and the run's output must be kept as `.` in the module's outputs objects, read back through +// its verb output. With steps.Install set (node-build-node alone), an install ask must end passed naming its archive, the +// install body stood in by the kit's. +func SeatVerbs(t *testing.T, seat string, tools []stdio.Tool, steps steprun.AskSteps) { + t.Helper() + for _, p := range JudgeSeatVerbs(seat, tools, steps) { + t.Error(p) + } +} + +// The kit's pipeline step: its text, its failure, its test not run and its archive. +const ( + kitText = "=== RUN TestKit\n kit_test.go:7: got 1, want 2\n--- FAIL: TestKit (0.00s)\n" + kitArchive = "oci-artifact-registry://kit/node-modules:0.1.1" + kitCommit = "0123456789abcdef0123456789abcdef01234567" +) + +var ( + kitNotRun = buildstep.NotRun{Test: "kit/m TestBwrap", Reason: "no bwrap where this check runs"} + kitFailure = buildstep.Failure{Package: "kit/m", Test: "TestKit", Cause: "kit_test.go:7: got 1, want 2"} +) + +// JudgeSeatVerbs is what SeatVerbs fails a build module for: nothing when its seat verbs and steps keep the part's form 2. +func JudgeSeatVerbs(seat string, tools []stdio.Tool, steps steprun.AskSteps) []string { + var problems []string + bad := func(format string, args ...any) { problems = append(problems, fmt.Sprintf(format, args...)) } + language, err := buildstep.AskLanguageOfSeat(seat) + if err != nil { + return []string{err.Error()} + } + tool := func(verb string) *stdio.Tool { + for i := range tools { + if tools[i].Name == seat+"."+verb { + return &tools[i] + } + } + bad("the seat tools serve no %s.%s", seat, verb) + return nil + } + current, output, answer := tool("current"), tool(buildstep.OutputVerb), tool(buildstep.AnswerVerb) + if current != nil { + problems = append(problems, judgeCurrent(*current, steps.Node)...) + } + if answer != nil && !slices.Equal(required(*answer), []string{"ask", "call"}) { + bad("%s.answer takes %v, want the ask and the call of its chunks", seat, required(*answer)) + } + if output != nil { + if req := required(*output); !slices.Equal(req, []string{"ask"}) || hasProperty(*output, "call") { + bad("%s.output takes %v, want the ask alone (form 2: no call, no chunks)", seat, req) + } + if got, err := output.Run(map[string]any{"ask": buildstep.NewID()}); err == nil { + bad("%s.output answers an ask never run here (%v), want it refused", seat, got) + } + } + if steps.Outputs == nil { + bad("the steps keep no outputs (AskSteps.Outputs): a run's output is never kept") + } + problems = append(problems, judgeStep(seat, language, steps, output)...) + if steps.Install != nil { + problems = append(problems, judgeInstall(seat, steps)...) + } + return problems +} + +// judgeCurrent runs current and judges its answer's form 2 fields. +func judgeCurrent(current stdio.Tool, node string) []string { + got, err := current.Run(map[string]any{}) + if err != nil { + return []string{fmt.Sprintf("%s: %v", current.Name, err)} + } + var c map[string]json.RawMessage + if raw, err := json.Marshal(got); err != nil || json.Unmarshal(raw, &c) != nil { + return []string{fmt.Sprintf("%s answers no object: %v", current.Name, got)} + } + var problems []string + for _, k := range []string{"node", "takes", "since", "capacity", "asks"} { + if _, ok := c[k]; !ok { + problems = append(problems, fmt.Sprintf("%s answers no %s (form 2): %v", current.Name, k, got)) + } + } + var n string + if json.Unmarshal(c["node"], &n); n != node { + problems = append(problems, fmt.Sprintf("%s names the node %q, want %q", current.Name, n, node)) + } + var asks []map[string]any + _ = json.Unmarshal(c["asks"], &asks) + for _, a := range asks { + if a["asker"] == nil { + problems = append(problems, fmt.Sprintf("%s answers an ask with no asker: %v", current.Name, a)) + } + } + return problems +} + +// judgeStep runs the kit's pipeline step through steps and judges its end and its kept output. +func judgeStep(seat, language string, steps steprun.AskSteps, output *stdio.Tool) []string { + var problems []string + bad := func(format string, args ...any) { problems = append(problems, fmt.Sprintf(format, args...)) } + var archives []any + var ends []buildstep.StepEnded + steps.Fetch = func(context.Context, map[string]any) (any, error) { + return map[string]any{"workspace": "/kit", "commit": kitCommit, "modules": []string{"modules/kit"}}, nil + } + steps.Digest, steps.Toolchain, steps.Kept, steps.Release, steps.Has = nil, nil, nil, nil, nil + steps.Test = func(_ context.Context, args map[string]any) (any, error) { + archives = append(archives, args["archive"]) + return map[string]any{"passed": false, "tests": 2, "why": "1 failed", "output": kitText, + "not-run": []buildstep.NotRun{kitNotRun}, "failed": []buildstep.Failure{kitFailure}}, nil + } + steps.SayStep = func(e buildstep.StepEnded) error { ends = append(ends, e); return nil } + id := buildstep.NewID() + ask := buildstep.StepAsk{ID: id, Kind: buildstep.AskStep, Language: language, Asker: "mesh-delivery-pipeline", At: time.Now(), + Step: &buildstep.StepRun{Pipeline: "main.1.kit", Step: buildstep.StepUnit, Module: "kit", Artifact: "kit:program", Commit: kitCommit, + Fingerprint: "fp-kit", Archive: kitArchive, Fetch: buildstep.Fetch{Repository: "novox/kit", Commit: kitCommit, Dir: "modules/kit"}}} + if _, err := steprun.RunAsk(context.Background(), steps, ask); err != nil { + return []string{fmt.Sprintf("the kit's pipeline step: %v", err)} + } + if len(ends) != 1 { + return []string{fmt.Sprintf("the kit's pipeline step said %d ends, want one", len(ends))} + } + e := ends[0] + if e.Outcome != buildstep.OutcomeFailed || !slices.Equal(e.NotRun, []buildstep.NotRun{kitNotRun}) { + bad("the kit's pipeline step ended %s with not-run %+v, want failed with %+v", e.Outcome, e.NotRun, kitNotRun) + } + if len(archives) == 0 || slices.ContainsFunc(archives, func(a any) bool { return a != kitArchive }) { + bad("the step's archive %s did not reach every test call: %v", kitArchive, archives) + } + if output == nil { + return problems + } + got, err := output.Run(map[string]any{"ask": id}) + raw, _ := json.Marshal(got) + var o buildstep.OutputAnswer + if err != nil || json.Unmarshal(raw, &o) != nil { + return append(problems, fmt.Sprintf("%s.output of the step run (%s): %v, %v; want its output kept as . in the outputs objects", seat, id, got, err)) + } + want := []buildstep.FailingTest{{Test: "kit/m TestKit", Words: kitFailure.Cause}} + if o.Ask != id || o.Output != kitText || o.Size != len(kitText) || !slices.Equal(o.Failing, want) || o.NoLongerKept { + bad("%s.output of the step run answers %+v, want the ask, its text, its size %d and failing %+v", seat, o, len(kitText), want) + } + return problems +} + +// judgeInstall runs an install ask through steps, the install body stood in by the kit's, and judges its end. +func judgeInstall(seat string, steps steprun.AskSteps) []string { + if seat != "node-build-node" { + return []string{fmt.Sprintf("%s installs (AskSteps.Install set): only node-build-node accepts the kind install", seat)} + } + var ends []buildstep.StepEnded + steps.Install = func(_ context.Context, args map[string]any) (any, error) { + if args["lock"] != "{}" || args["artifact"] != buildstep.InstallArtifact("kit") || args["version"] != "0.1.1" { + return nil, fmt.Errorf("the install was asked %v", args) + } + return map[string]any{"archive": kitArchive}, nil + } + steps.SayStep = func(e buildstep.StepEnded) error { ends = append(ends, e); return nil } + id := buildstep.NewID() + ask := buildstep.StepAsk{ID: id, Kind: buildstep.AskInstall, Language: buildstep.AskNode, Asker: "mesh-delivery-pipeline", At: time.Now(), Version: "0.1.1", + Install: &buildstep.InstallAsk{ID: id, Asker: "mesh-delivery-pipeline", Pipeline: "main.1.kit", Module: "kit", Commit: kitCommit, Fingerprint: "fp-kit", Lock: "{}"}} + if _, err := steprun.RunAsk(context.Background(), steps, ask); err != nil { + return []string{fmt.Sprintf("the kit's install: %v", err)} + } + if len(ends) != 1 || ends[0].Outcome != buildstep.OutcomePassed || ends[0].Archive != kitArchive { + return []string{fmt.Sprintf("the kit's install said %+v, want one end passed naming the archive %s", ends, kitArchive)} + } + return nil +} + +// required is the sorted names a tool's input requires. +func required(t stdio.Tool) []string { + var out []string + switch r := t.Input["required"].(type) { + case []string: + out = append(out, r...) + case []any: + for _, s := range r { + if s, ok := s.(string); ok { + out = append(out, s) + } + } + } + slices.Sort(out) + return out +} + +// hasProperty says whether a tool's input names the property name. +func hasProperty(t stdio.Tool, name string) bool { + props, _ := t.Input["properties"].(map[string]any) + _, ok := props[name] + return ok +}