diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index fdaf3057..26e4fd35 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -149,8 +149,13 @@ type stepSeenKey struct{} // WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node // that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so -// (issue 1890). seen runs in the waiting goroutine and must not block. +// (issue 1890). seen runs in the waiting goroutine and must not block. A seen ctx already carries is kept: both are handed +// every value, the earlier first (a check's begin bound watches the steps of a try that names them). func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context { + if before := stepSeenOf(ctx); before != nil { + later := seen + seen = func(s delivery.Step) { before(s); later(s) } + } return context.WithValue(ctx, stepSeenKey{}, seen) } diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index ec6b94f9..3011500d 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -26,10 +26,13 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)} }} k := Asker{Publisher: f, Waiter: f} - var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it - step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957")) - if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" { - t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen) + var seen, also []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it + // Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first. + both := WithStepSeen(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), + func(s delivery.Step) { also = append(also, s) }) + step, err := k.Ask(both, goAsk("mvl0frv18d1957")) + if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" || len(also) != 1 { + t.Fatalf("step %+v, %v, asks %d, seen %+v and %+v", step, err, len(f.Asks()), seen, also) } f.Answer = func(a delivery.StepAsk) delivery.Step { return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"} diff --git a/modules/mesh-delivery-checker/cmd/mesh-checking-program/main.go b/modules/mesh-delivery-checker/cmd/mesh-checking-program/main.go index 21279f08..a55d209b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-checking-program/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-checking-program/main.go @@ -4,16 +4,18 @@ // verdict the judging rules give them, as one JSON line on its standard output (rules.Answer). // // mesh-checking-program -verdict +// mesh-checking-program -seats // +// Asked -seats, it answers the seat list of the heads laid in the directory MESH_RUN_HEADS names (rules.SeatsAnswer). // It refuses facts of a version it does not read, naming it, with exit 2. package main import ( - "encoding/json" - "flag" "fmt" "io" + "io/fs" "os" + "slices" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-checker/rules" @@ -25,52 +27,25 @@ func main() { // run is the command over its arguments, environment and outputs, answering its exit. func run(args []string, getenv func(string) string, stdout, stderr io.Writer) int { - fs := flag.NewFlagSet("mesh-checking-program", flag.ContinueOnError) - fs.SetOutput(stderr) - verdict := fs.Bool("verdict", false, "answer the verdict of the facts in the input") - seats := fs.Bool("seats", false, "answer the seat list of the heads laid for it") - if err := fs.Parse(args); err != nil { + var heads fs.FS + if dir := getenv(delivery.RunHeadsEnv); dir != "" { + heads = os.DirFS(dir) + } + input := func() ([]byte, error) { + file := getenv(delivery.RunInputEnv) + if file == "" { + return nil, fmt.Errorf("%s names no file", delivery.RunInputEnv) + } + return os.ReadFile(file) + } + line, err := rules.Program(args, input, heads) + if err != nil { + if heads == nil && slices.Contains(args, "-seats") { + err = fmt.Errorf("no heads: %s names no directory", delivery.RunHeadsEnv) + } + fmt.Fprintf(stderr, "mesh-checking-program: %v\n", err) return 2 } - refuse := func(format string, a ...any) int { - fmt.Fprintf(stderr, "mesh-checking-program: "+format+"\n", a...) - return 2 - } - if *seats { - dir := getenv(delivery.RunHeadsEnv) - if dir == "" { - return refuse("no heads: %s names no directory", delivery.RunHeadsEnv) - } - answer, err := rules.SeatsOf(os.DirFS(dir)) - if err != nil { - return refuse("%v", err) - } - line, err := json.Marshal(answer) - if err != nil { - return refuse("the seat list cannot be written: %v", err) - } - fmt.Fprintf(stdout, "%s\n", line) - return 0 - } - if !*verdict { - return refuse("nothing asked: give -verdict or -seats") - } - file := getenv(delivery.RunInputEnv) - if file == "" { - return refuse("no input: %s names no file", delivery.RunInputEnv) - } - data, err := os.ReadFile(file) - if err != nil { - return refuse("the input cannot be read: %v", err) - } - facts, err := rules.DecodeFacts(data) - if err != nil { - return refuse("%v", err) - } - line, err := json.Marshal(rules.Judged(facts)) - if err != nil { - return refuse("the verdict cannot be written: %v", err) - } fmt.Fprintf(stdout, "%s\n", line) return 0 } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index a996d06c..2e29ae40 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -264,6 +264,17 @@ func Tools(r check.Reader) []stdio.Tool { kept, cut, err := r.Kept(level, digest) return cutAnswer(map[string]any{"kept": kept}, cut), err }}, + {Name: check.InputsReadTool, + Description: "The checking program's inputs the checker keeps, each under the sha256 of its exact bytes (inputs., kept a day): " + + "the facts of a check its verdict is asked of, or the heads its seat list is asked of. With a digest, that input as kept; " + + "without, every digest and its size. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "digest": map[string]any{"type": "string", "description": "the input's sha256, as a run ask names it"}}}, + Run: func(args map[string]any) (any, error) { + digest, _ := args["digest"].(string) + inputs, cut, err := r.Inputs(digest) + return cutAnswer(map[string]any{"inputs": inputs}, cut), err + }}, {Name: check.PreviewsReadTool, Description: "The merge previews the checker holds, each as stored under previews...: the merge commit, per module directory " + "its tree, manifest digest and lock hashes, or the files in conflict. With a repository, that repository's; with a head (7 or more characters) " + @@ -325,9 +336,9 @@ func stepBound() time.Duration { func main() { r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")), - PreviewState: read(stdio.State("previews")), - StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, - HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(), + PreviewState: read(stdio.State("previews")), InputState: read(stdio.State(check.InputsStateName)), + StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, + HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(), TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)), AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))} r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 08aa1ca0..3ef71945 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -21,7 +21,7 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) { t.Errorf("%s has no description or no run", tl.Name) } } - want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool} + want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.InputsReadTool, check.PreviewsReadTool, check.StatusReadTool} if !slices.Equal(names, want) { t.Errorf("tools = %v, want %v", names, want) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go index 62eccf09..4c3dcd3c 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go @@ -183,3 +183,15 @@ func (a awaitedKV) Delete(ctx context.Context, key string) error { } return s.Delete(ctx, key) } + +// runInputs keeps the checking program's inputs in the checker's state inputs (check.InputsBucket), each under the +// sha256 of its exact bytes: an input already kept under its digest is the same bytes, so a second keep is no error. +type runInputs struct{ a awaitedKV } + +func (r runInputs) Keep(ctx context.Context, digest string, body []byte) error { + _, err := r.a.Create(ctx, digest, body) + if errors.Is(err, check.ErrConflict) { + return nil + } + return err +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 72cd4811..863eb35d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -24,6 +24,8 @@ import ( // judgeDeps are the parts the judge of a check is made of. type judgeDeps struct { + // inputs keeps the checking program's inputs (check.InputsState). + inputs check.Inputs asks check.Asks tools *buildrun.ToolchainSet trunks *check.Trunks @@ -160,7 +162,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Inputs: d.inputs, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go index 23f32299..fb84bd24 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go @@ -38,7 +38,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { // t.Errorf("announced %v, the manifest keeps %v", announced, manifest) } // kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage). - if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) { + // inputs by the Go build module, which hands one to the checking program a run ask names (issue 1921). + if want := []string{"inputs", "kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) { t.Errorf("readable %v, want %v", readable, want) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 877f3627..8dfe2f28 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -72,6 +72,9 @@ var keptStates = []announce.State{ {Name: "hearing"}, // The asks each running check put (check.AsksState): read by the checker's own checker_ask_again alone. {Name: "asks"}, + // The checking program's inputs by their digest (check.InputsState): read by the Go build module, which hands one to + // the program a run ask names (issue 1921). + {Name: check.InputsStateName, Readable: true}, {Name: "kept", Readable: true}, {Name: "owed", Readable: true}, {Name: "previews", Readable: true}, @@ -153,6 +156,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") }) waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing} asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf) + inputsKV := process.AwaitBucket(ctx, js, conn, check.InputsBucket, "no checking program can be asked, so every check fails naming it", logger.Printf) ch := &check.Checker{ Asks: awaitedKV{asksKV}, Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]}, @@ -226,7 +230,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]}, repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, - announce: announce, compositions: compositions, + announce: announce, compositions: compositions, inputs: runInputs{awaitedKV{inputsKV}}, nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState, "a change's manifests are judged with no predefined keys, each fallback as its value declares it", logger.Printf)}, diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 050bd601..ba8c72da 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -1,6 +1,7 @@ package check import ( + "encoding/json" "fmt" "maps" "slices" @@ -39,6 +40,9 @@ type Case struct { Left []string // Queued is when the checker took the case up, for the time of its queue. Queued time.Time + // Seats is the heads' seat list the checking program made, which the manifest validator's and the node-engine + // composer's requests carry (ADR 0636 §4); none until it is made. + Seats []json.RawMessage // Try narrows a try's check (its level, its tests); nil for a pull request's. Try *TryOptions diff --git a/modules/mesh-delivery-checker/internal/check/engines.go b/modules/mesh-delivery-checker/internal/check/engines.go index f0cadd76..d6159d0e 100644 --- a/modules/mesh-delivery-checker/internal/check/engines.go +++ b/modules/mesh-delivery-checker/internal/check/engines.go @@ -171,7 +171,7 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s } } req := composer.Request{V: composer.Version, Input: a.Input, Depth: composer.DepthCompose, Machines: []string{node}, - Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}} + Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}, Seats: c.Seats} raw, err := json.Marshal(req) if err != nil { return a, err diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index 189fedb5..2157321c 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -146,6 +146,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { t.Fatal(err) } host := CaseOf(c, map[string]string{repoA: trunkA, nodeEngineRepo: trunkA}) + host.Seats = []json.RawMessage{json.RawMessage(`{"seat":"node-x","scope":"node"}`)} // the heads' list (ADR 0636 §4) withInput := composed("anchor") withInput.Input = json.RawMessage(`{"nodes":[{"name":"anchor"}]}`) e := &fakeEngines{answers: map[string]delivery.ComposeCandidateAnswer{"anchor": withInput}} @@ -167,7 +168,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { var req composer.Request _ = json.Unmarshal(job.Request, &req) if job.Fetch["repository"] != nodeEngineRepo || job.Fetch["commit"] != head2 || job.Package != composer.Package || - string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" { + string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" || + len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) { t.Errorf("job = %+v, request %+v", job, req) } // Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet. diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 2fa8dab3..efa9362d 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1,12 +1,15 @@ package check import ( + "cmp" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "io/fs" + "mesh-delivery-checker/rules" "os" "path/filepath" "reflect" @@ -15,6 +18,7 @@ import ( "strings" "sync" "testing" + "testing/fstest" "time" "unicode/utf8" @@ -125,6 +129,69 @@ type bench struct { // declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes // one (issue 890); an ask naming none runs on shanks, which then has none. declines int + // inputs are the checking program's kept inputs; heads the heads laid for it (none when nil); run, when set, answers + // a run ask in its place (a program that does not build or answer). + inputs *memKV + heads fs.FS + run func(a delivery.StepAsk) (delivery.Step, bool) + mu sync.Mutex + runs []delivery.StepAsk // the run asks put +} + +// benchAsks puts the bench's asks: a run ask is answered at once by the checking program's work (answerRun), written by +// write and kept apart (bench.runs); every other ask goes to inner (the fake bus by default), answered as it answers. +type benchAsks struct { + b *bench + inner buildrun.AskPublisher + write func(delivery.Step) +} + +func (b *bench) asker(inner buildrun.AskPublisher, write func(delivery.Step)) benchAsks { + return benchAsks{b: b, inner: inner, write: write} +} + +func (p benchAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error { + inner, write := p.inner, p.write + if inner == nil { + inner, write = p.b.asks, p.b.asks.Write + } + a, err := delivery.DecodeStepAsk(data) + if err != nil || a.Run == nil { + return inner.PublishAsk(ctx, subject, id, data) + } + p.b.mu.Lock() + p.b.runs = append(p.b.runs, a) + p.b.mu.Unlock() + if step := p.b.answerRun(a); step.ID != "" { + write(step) + } + return nil +} + +// trunkD is mesh-delivery's trunk the bench's checking program is built at. +var trunkD = strings.Repeat("d", 40) + +// answerRun answers a run ask as the build module would: the checking program's work (rules.Program) over the input the +// ask names, read from the kept inputs and checked against its digest. +func (b *bench) answerRun(a delivery.StepAsk) delivery.Step { + began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) + step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "shanks", State: delivery.StepPassed, Started: began, Ended: began.Add(time.Second)} + if b.run != nil { + if s, ok := b.run(a); ok { + return s + } + } + answer := delivery.RunAnswer{Built: true, Key: "program-0123456789abcdef"} + out, err := rules.Program(a.Run.Args, func() ([]byte, error) { + body, _, _, _ := b.inputs.Get(context.Background(), a.Run.Input.Digest) + return body, a.Run.Input.Verify(body) + }, b.heads) + if err != nil { + answer.Exit, answer.Said = 2, err.Error() + } + answer.Output = out + step.Result, _ = json.Marshal(answer) + return step } // refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them. @@ -183,7 +250,7 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step { // newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass // the captured slow fixture's result at both levels, on shanks. func newBench(t testing.TB) *bench { - b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV()} + b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV(), inputs: newMemKV(), heads: fstest.MapFS{}} b.fetch = captured[Fetched](t, "go-fetch-clean") b.fetch.Modules, b.fetch.Changed, b.fetch.Trees = []string{"."}, []string{"README.md"}, nil one := captured[Digested](t, "go-digest") @@ -207,8 +274,8 @@ func newBench(t testing.TB) *bench { } now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) b.asks = &buildrun.FakeAsks{Answer: b.answer} - b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev, - Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }} + b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev, + Modules: langs{}, Trunks: &Trunks{heads: map[string]string{DeliveryRepository: trunkD}}, Inputs: memInputs{b.inputs}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }} return b } @@ -299,6 +366,73 @@ func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679 t.Errorf("the check ask carries the names %v", asks) } } + + // Issue 1921 (B0, D4 to D6): the check's verdict is the checking program's, built at the group's mesh-delivery head, + // else at mesh-delivery's trunk, over the facts kept as its input; a program that is never begun, does not build or + // does not answer fails the check naming it, and a rule the change's program alters judges the check. + verdictOf := func(state, words string) json.RawMessage { + out, _ := json.Marshal(rules.Answer{V: rules.AnswerVersion, Verdict: delivery.Verdict{State: state, Failed: delivery.FailedFirst{Words: words}}}) + return out + } + for _, row := range []struct { + name string + repo string // the member's repository: mesh-delivery unless named + run func(a delivery.StepAsk, s delivery.Step) (delivery.Step, bool) + at string // the commit the program is built at; empty when it fails + fails string + runs int // the run asks put when it fails: each put twice when never begun + }{ + {name: "at the group's mesh-delivery head", at: head1}, + {name: "at mesh-delivery's trunk", repo: "novox/mesh-catalog", at: trunkD}, + {name: "never begun", run: func(delivery.StepAsk, delivery.Step) (delivery.Step, bool) { return delivery.Step{}, true }, + fails: "no build node began the job run of the asks", runs: 4}, + {name: "does not build", run: func(_ delivery.StepAsk, s delivery.Step) (delivery.Step, bool) { + s.State, s.Reason = delivery.StepFailed, "the program does not build: undefined: rules.Nothing" + return s, true + }, fails: "by the checking program: its run ask", runs: 2}, + {name: "a rule the change alters", at: head1, run: func(a delivery.StepAsk, s delivery.Step) (delivery.Step, bool) { + if a.Run.Args[0] != "-verdict" { + return s, false + } + s.Result, _ = json.Marshal(delivery.RunAnswer{Key: "program-0123456789abcdef", Output: verdictOf(delivery.VerdictFail, "the change's own rule refuses it")}) + return s, true + }, fails: "the change's own rule refuses it"}, + } { + b := newBench(t) + b.ex.BeginBound = 20 * time.Millisecond + if row.run != nil { + b.run = func(a delivery.StepAsk) (delivery.Step, bool) { + return row.run(a, delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepPassed}) + } + } + repo := cmp.Or(row.repo, repoA) + v := b.judge(t, caseOf(t, pullOf(repo, 7, head1))).Verdict + if row.fails != "" { + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, row.fails) { + t.Errorf("%s: verdict %+v, want a fail saying %q", row.name, v, row.fails) + } + if row.at == "" && (len(b.runs) != row.runs || v.JudgedBy != nil) { + t.Errorf("%s: %d run asks, judged by %+v: want %d and none judging", row.name, len(b.runs), v.JudgedBy, row.runs) + } + } else if v.State != delivery.VerdictPass { + t.Errorf("%s: verdict %+v", row.name, v) + } + if row.at == "" { + continue + } + if by := v.JudgedBy; by == nil || by.Program.Commit != row.at || by.Program.Package != CheckingProgramPackage || by.Key == "" || + len(by.Heads) != 1 || by.Heads[0].Repository != repo || by.Heads[0].Commit != head1 { + t.Errorf("%s: judged by %+v, want the program at %s over the member's head", row.name, v.JudgedBy, row.at) + } + if len(b.runs) != 2 || b.runs[0].Run.Args[0] != "-seats" || b.runs[1].Run.Args[0] != "-verdict" { + t.Fatalf("%s: run asks %+v, want the seat list and then the verdict", row.name, b.runs) + } + in := b.runs[1].Run.Input // the facts kept as their exact bytes under their digest + if body, _, found, _ := b.inputs.Get(context.Background(), in.Digest); !found || in.Verify(body) != nil || in.State != InputsState || + !strings.Contains(string(body), `"group":"feat/x"`) { + t.Errorf("%s: the verdict's input %+v is not kept whole: %s", row.name, in, body) + } + } } // namesOf is the nodes' names as a test gives them. @@ -923,7 +1057,7 @@ func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) { b.asks.Answer = func(a delivery.StepAsk) delivery.Step { return delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()} } - b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond} + b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") { t.Fatalf("verdict = %+v", out.Verdict) @@ -959,7 +1093,7 @@ func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) { b := newBench(t) answer := b.asks.Answer b.asks.Answer = nil - b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond} + b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} done := make(chan Outcome, 1) go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }() for deadline := time.Now().Add(10 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) { @@ -1047,8 +1181,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's // group (issue 1921): the same judge, the same verdicts. try bool + // seats lays heads that add a seat: the checking program's seat list is carried by every judge ask's request (ADR + // 0636 §4, issue 1921 D4). + seats bool }{ {name: "accepted"}, + {name: "the heads' seat list is carried by the head's judge and the trunk's", seats: true, indexer: "ok", ownJudge: true}, {name: "accepted in a group try", try: true}, {name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`}, {name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"}, @@ -1069,6 +1207,9 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { {name: "refused", refused: "web requires \"amqp\"\nweb has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web requires "amqp"; web has no id`}, } { b := newBench(t) + if row.seats { + b.heads = fstest.MapFS{"novox/mesh-contracts/contracts/seats/node-x/seat.json": {Data: []byte(`{"format":1,"seat":"node-x","scope":"node"}`)}} + } b.ex.Manifests = fixedDigests{"mail": strings.Repeat("a", 64), "web": strings.Repeat("b", 64)} b.ex.Trunks = fixedTrunks{heads: map[string]string{DeliveryRepository: trunk}} b.ex.Predefined = keptList(`{"font-size": {"type": "number", "description": "the size"}}`) @@ -1174,6 +1315,13 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir { t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v) } + for _, j := range judged { + var seated composer.Request + if err := json.Unmarshal(j.Request, &seated); err != nil || (len(seated.Seats) == 1) != row.seats || + (row.seats && !strings.Contains(string(seated.Seats[0]), `"node-x"`)) { + t.Errorf("%s: the judge at %v is handed the seats %s, want the heads' list: %v", row.name, j.Fetch["commit"], seated.Seats, err) + } + } var req composer.Request if err := json.Unmarshal(judged[0].Request, &req); err != nil || req.Cases[0].Manifests["web"].Digest != web || req.Cases[1].Name != JudgeTrunk || req.Cases[1].Manifests["web"].Digest != strings.Repeat("b", 64) || diff --git a/modules/mesh-delivery-checker/internal/check/fakes_test.go b/modules/mesh-delivery-checker/internal/check/fakes_test.go index d53f2a14..bade10ab 100644 --- a/modules/mesh-delivery-checker/internal/check/fakes_test.go +++ b/modules/mesh-delivery-checker/internal/check/fakes_test.go @@ -68,3 +68,14 @@ func (m *memKV) Delete(_ context.Context, key string) error { delete(m.revs, key) return nil } + +// memInputs keeps run inputs in a memKV, under their digest. +type memInputs struct{ kv *memKV } + +func (m memInputs) Keep(ctx context.Context, digest string, body []byte) error { + if _, _, found, _ := m.kv.Get(ctx, digest); found { + return nil + } + _, err := m.kv.Create(ctx, digest, body) + return err +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 78856a58..036f98e8 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -138,6 +138,10 @@ type Examiner struct { // and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no // manifest or composition judged, said. Candidates TryCandidates + // Inputs keeps the checking program's inputs (the state InputsState); BeginBound is how long a run ask may wait for a + // build node to begin it (DefaultBeginBound when zero). + Inputs Inputs + BeginBound time.Duration Now func() time.Time Logf func(format string, args ...any) } @@ -302,13 +306,17 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { // Each failure is a fact recorded where it is met; the verdict is the judging rules' of the facts. st.judgeAll(ctx) st.takeTryCandidate(ctx) + // The heads' seat list, from the checking program, before the validator and the composer are asked (ADR 0636 §4). + st.seatList(ctx) st.judgeManifests(ctx) st.refuseCycle(ctx) st.compose(ctx) - answer := rules.Judged(st.factsOf()) + // The verdict is the checking program's of the facts (ADR 0644 §1, ADR 0656 §5). + answer, by := st.judged(ctx) for _, n := range answer.Notes { st.c.note(n) } + answer.Verdict.JudgedBy = by out := st.outcome(answer.Verdict) out.Builds = st.j.buildNotes(ctx, st.c) return out, nil @@ -384,7 +392,9 @@ func (st *state) fail(machine, words string) { // factsOf are the check's facts as met: what the checking program judges. func (st *state) factsOf() rules.Facts { - return rules.Facts{V: rules.FactsVersion, Steps: st.facts, CloneMS: st.clone.Milliseconds(), Composition: st.composition, Notes: st.c.Notes()} + f := st.factsHead() + f.Steps, f.CloneMS, f.Composition, f.Notes = st.facts, st.clone.Milliseconds(), st.composition, st.c.Notes() + return f } // levels are the levels a case's asks run: both, or the one a try names. diff --git a/modules/mesh-delivery-checker/internal/check/lostask_test.go b/modules/mesh-delivery-checker/internal/check/lostask_test.go index 3c547e46..d84a58af 100644 --- a/modules/mesh-delivery-checker/internal/check/lostask_test.go +++ b/modules/mesh-delivery-checker/internal/check/lostask_test.go @@ -58,7 +58,7 @@ func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data [] func losing(b *bench, lose int) *losingAsks { l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose} - b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second} + b.ex.Asks = buildrun.Asker{Publisher: b.asker(l, l.FakeAsks.Write), Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second} return l } @@ -282,7 +282,7 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) { tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf, deafIDs: map[string]bool{}, steps: map[string]delivery.Step{}} heard = &buildrun.HeardEnds{Inner: tr, Read: tr} - b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second} + b.ex.Asks = buildrun.Asker{Publisher: b.asker(tr, tr.Write), Waiter: heard, Bound: 5 * time.Second} asked = func() int { return len(tr.Asks()) } } r := newRig(t, b.ex) diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index 718cebd5..90bba199 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -123,6 +123,8 @@ func (st *state) judgeManifests(ctx context.Context) { req.Predefined = list st.c.note("the manifests are judged against the predefined keys of " + from) } + // The heads' seat list, for the change case and the trunk case both (ADR 0636 §4). + req.Seats = st.c.Seats cv, trunkCase, ok := st.runJudge(ctx, req, touched, commit, at, st.packagesOf(DeliveryRepository)) if !ok { return diff --git a/modules/mesh-delivery-checker/internal/check/programs.go b/modules/mesh-delivery-checker/internal/check/programs.go new file mode 100644 index 00000000..1b5b9cb4 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/programs.go @@ -0,0 +1,223 @@ +package check + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "mesh-delivery-checker/rules" +) + +// The checking program (novox/hq ADR 0644 §1, ADR 0656 §5; design 47's bootstrap delivery; issue 1921): every check has +// the program built from the group's mesh-delivery member head, else from mesh-delivery's trunk, and run in the build +// seat's sandbox through the seat node-build-go's ask `run`, with every member's head laid for it. It answers the heads' +// seat list first (`-seats`), which the manifest validator's and the composer's requests carry, and then the check's +// verdict of the facts the asking gathered (`-verdict`). The running checker asks, keeps and records; it judges nothing. + +const ( + // InputsStateName is the checker's own state of run inputs, InputsBucket its bucket and InputsState its name as a run + // ask's input names it (delivery.RunInput.State): every input by the sha256 of its exact bytes, read by the Go build + // module, which checks the digest (delivery.RunInput.Verify). + InputsStateName = "inputs" + InputsBucket = delivery.CheckerModule + "_" + InputsStateName + InputsState = delivery.CheckerModule + "." + InputsStateName + // CheckingProgramDir and CheckingProgramPackage are where in mesh-delivery the checking program is built. + CheckingProgramDir = "modules/mesh-delivery-checker" + CheckingProgramPackage = "mesh-delivery-checker/cmd/mesh-checking-program" + // DefaultBeginBound is how long a run ask may wait for a build node to begin its step before it is put once more, and + // after that once more before the check fails naming the job: a build module that does not know the job `run` drops + // it unanswered (the lead's ruling on milestone 47/B0, 2026-11-06). + DefaultBeginBound = 5 * time.Minute +) + +// Inputs keeps a run's input under its digest, exactly its bytes. +type Inputs interface { + Keep(ctx context.Context, digest string, body []byte) error +} + +func (j *Examiner) beginBound() time.Duration { + if j.BeginBound > 0 { + return j.BeginBound + } + return DefaultBeginBound +} + +// checkingProgram is where this check's checking program is built: the group's mesh-delivery member head, else +// mesh-delivery's trunk; false when neither is known. +func (st *state) checkingProgram() (delivery.RunProgram, string, bool) { + p := delivery.RunProgram{Repository: DeliveryRepository, Dir: CheckingProgramDir, Package: CheckingProgramPackage} + for _, pull := range st.c.Pulls { + if pull.Repository == DeliveryRepository { + p.Commit = pull.Head + return p, "the group's mesh-delivery head " + pull.Head, true + } + } + if st.j.Trunks != nil { + p.Commit = st.j.Trunks.Heads()[DeliveryRepository] + } + return p, "mesh-delivery's trunk " + p.Commit, p.Commit != "" +} + +// runHeads are the members' heads, as the program reads them. +func (st *state) runHeads() []delivery.RunHead { + out := make([]delivery.RunHead, 0, len(st.c.Pulls)) + for _, p := range st.c.Pulls { + out = append(out, delivery.RunHead{Repository: p.Repository, Commit: p.Head}) + } + return out +} + +// runProgram has the checking program built and run with args over body, and answers its output and what judged; an +// error says why it did not answer: its input over its bound, no build node beginning the job, a step that did not +// pass, or a program that exited otherwise than 0. +func (st *state) runProgram(ctx context.Context, args []string, body []byte) (json.RawMessage, *delivery.JudgedBy, error) { + prog, at, ok := st.checkingProgram() + if !ok { + return nil, nil, fmt.Errorf("it cannot be built: the trunk head of %s is not known", DeliveryRepository) + } + if st.j.Inputs == nil { + return nil, nil, fmt.Errorf("its input cannot be kept: the checker keeps no state %s", InputsState) + } + in, err := delivery.RunInputOf(InputsState, body) + if err != nil { + return nil, nil, err + } + if err := st.j.Inputs.Keep(ctx, in.Digest, body); err != nil { + return nil, nil, fmt.Errorf("its input could not be kept in %s: %w", InputsState, err) + } + job := delivery.RunJob{Program: prog, Heads: st.runHeads(), Input: in, Args: args, Reader: "the checking program at " + at} + var ids []string + for range 2 { + a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule, + At: st.j.now().UTC(), Packages: st.packagesOf(DeliveryRepository), Run: &job} + ids = append(ids, a.ID) + st.c.Progress.Set(fmt.Sprintf("the run ask %s of the checking program %s", a.ID, strings.Join(args, " ")), "") + st.c.Asked(a.ID) + if err := st.j.Asks.Put(ctx, a); err != nil { + return nil, nil, fmt.Errorf("its run ask could not be put: %w", err) + } + waiting := ctx + if watch := st.c.Watch; watch != nil { + id := a.ID + watch(id, nil) // a try names it at once (issue 1890) + waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) }) + } + step, begun, err := st.waitBegun(waiting, a) + if !begun { + continue + } + if err != nil { + return nil, nil, fmt.Errorf("its run ask %s: %w", a.ID, err) + } + if step.State != delivery.StepPassed { + return nil, nil, fmt.Errorf("its run ask %s ended %s on %s: %s", a.ID, step.State, step.Node, step.Reason) + } + var answer delivery.RunAnswer + if err := json.Unmarshal(step.Result, &answer); err != nil { + return nil, nil, fmt.Errorf("its run ask %s answered no run answer: %w", a.ID, err) + } + if answer.Exit != 0 { + return nil, nil, fmt.Errorf("it exited %d on %s: %s", answer.Exit, step.Node, answer.Said) + } + return answer.Output, &delivery.JudgedBy{Program: prog, Key: answer.Key, Built: answer.Built, Heads: job.Heads}, nil + } + return nil, nil, fmt.Errorf("no build node began the job run of the asks %s within %s each: a build module that does not know the job run drops it", + strings.Join(ids, " and "), st.j.beginBound()) +} + +// waitBegun waits for a's step, and stops waiting, begun false, when no build node began it within the begin bound: a +// bound on the wait's context, no timer of its own. +func (st *state) waitBegun(ctx context.Context, a delivery.StepAsk) (delivery.Step, bool, error) { + wait, cancel := context.WithCancel(ctx) + defer cancel() + begin, stop := context.WithTimeout(ctx, st.j.beginBound()) + defer stop() + seen := make(chan struct{}) + var once sync.Once + type ended struct { + step delivery.Step + err error + } + done := make(chan ended, 1) + go func() { + step, err := st.j.Asks.Wait(buildrun.WithStepSeen(wait, func(delivery.Step) { once.Do(func() { close(seen) }) }), a) + done <- ended{step, err} + }() + select { + case e := <-done: + return e.step, true, e.err + case <-seen: + case <-begin.Done(): + if ctx.Err() != nil { + return delivery.Step{}, true, ctx.Err() + } + return delivery.Step{}, false, nil + } + e := <-done + return e.step, true, e.err +} + +// seatList asks the checking program for the heads' seat list, which the validator's and the composer's requests carry +// (ADR 0636 §4); the list and what it left out are the check's notes. A list it cannot make fails the check, naming why. +func (st *state) seatList(ctx context.Context) { + if st.narrowTry() { + return + } + body, err := json.Marshal(st.factsHead()) + if err == nil { + var out json.RawMessage + if out, _, err = st.runProgram(ctx, []string{"-seats"}, body); err == nil { + var a rules.SeatsAnswer + if err = json.Unmarshal(out, &a); err == nil && a.V != rules.SeatsVersion { + err = fmt.Errorf("it answered a seat list of version %d, not %d", a.V, rules.SeatsVersion) + } + if err == nil { + st.c.Seats = a.Seats + st.c.note(fmt.Sprintf("the heads' seat list holds %d seat definitions", len(a.Seats))) + for _, l := range a.Left { + st.c.note("the heads' seat list leaves out " + l.Path + ": " + l.Why) + } + return + } + } + } + st.fail(goModule, "the heads' seat list could not be made by the checking program: "+err.Error()) +} + +// factsHead are the facts that name the check, without what the asking gathered. +func (st *state) factsHead() rules.Facts { + f := rules.Facts{V: rules.FactsVersion, Group: st.c.Group, Heads: st.c.Heads, Trunk: st.c.Trunk} + for _, p := range st.c.Pulls { + f.Members = append(f.Members, rules.Member{Repository: p.Repository, Number: p.Number, Head: p.Head, Trunk: p.Trunk}) + } + return f +} + +// judged is the checking program's answer to the check's facts (`-verdict`), with what judged; a program that cannot be +// asked, built or run, or does not answer a verdict, fails the check naming the checking program, never the running +// checker's rules in its place: the facts gain the failure and the verdict is worded from them as a failure. +func (st *state) judged(ctx context.Context) (rules.Answer, *delivery.JudgedBy) { + body, err := json.Marshal(st.factsOf()) + var by *delivery.JudgedBy + if err == nil { + var out json.RawMessage + if out, by, err = st.runProgram(ctx, []string{"-verdict"}, body); err == nil { + var a rules.Answer + if err = json.Unmarshal(out, &a); err == nil && a.V != rules.AnswerVersion { + err = fmt.Errorf("it answered a verdict of version %d, not %d", a.V, rules.AnswerVersion) + } + if err == nil { + return a, by + } + } + } + st.fail(goModule, "the check could not be judged by the checking program: "+err.Error()) + return rules.Answer{V: rules.AnswerVersion, Verdict: rules.Judge(st.factsOf())}, by +} diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index af42b94c..3457cd47 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -20,6 +20,7 @@ const ( VerdictsReadTool = "checker_verdicts" PullReadTool = "checker_pull" KeptReadTool = "checker_kept" + InputsReadTool = "checker_inputs" // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g). PreviewsReadTool = "checker_previews" StatusReadTool = "checker_status" @@ -36,6 +37,7 @@ type Reader struct { VerdictState, KeptState, CandidateState StateRead PreviewState StateRead // the checker's merge previews + InputState StateRead // the checking program's inputs (InputsState) StepStates map[string]StateRead // the build modules' steps states, by module HaveState StateRead // the node-engine's have: whether a build module is on a node and running StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 @@ -257,6 +259,42 @@ func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, return out, cut, nil } +// Inputs are the checking program's kept inputs: with a digest that input as kept, else every digest with its size. +func (r Reader) Inputs(digest string) (map[string]json.RawMessage, string, error) { + if r.InputState.Keys == nil || r.InputState.Get == nil { + return nil, "", fmt.Errorf("the state %s cannot be read here", InputsState) + } + if digest != "" { + raw, err := r.InputState.Get(digest) + if err != nil { + return nil, "", err + } + if len(raw) == 0 { + return nil, "", fmt.Errorf("no input %s is kept", digest) + } + return map[string]json.RawMessage{digest: raw}, "", nil + } + keys, err := r.InputState.Keys() + if err != nil { + return nil, "", err + } + sort.Strings(keys) + keys, cut := capKeys(keys, "give a digest to read one") + sizes := make([]int, len(keys)) + if err := parallel(len(keys), func(i int) error { + raw, err := r.InputState.Get(keys[i]) + sizes[i] = len(raw) + return err + }); err != nil { + return nil, "", err + } + out := map[string]json.RawMessage{} + for i, k := range keys { + out[k] = json.RawMessage(fmt.Sprintf(`{"size":%d}`, sizes[i])) + } + return out, cut, nil +} + // PreviewRead is one preview as stored, with the parts of its key. type PreviewRead struct { Key string `json:"key"` diff --git a/modules/mesh-delivery-checker/internal/check/read_test.go b/modules/mesh-delivery-checker/internal/check/read_test.go index 033ba635..c01589be 100644 --- a/modules/mesh-delivery-checker/internal/check/read_test.go +++ b/modules/mesh-delivery-checker/internal/check/read_test.go @@ -114,6 +114,21 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // if _, _, err := rd.Kept("unit", digB); err == nil { t.Error("a digest nothing kept was answered") } + // Issue 1921: checker_inputs answers the checking program's inputs, every digest with its size, or one as kept. + inputs := newMemKV() + body := []byte(`{"v":1,"group":"feat/x"}`) + in, _ := delivery.RunInputOf(InputsState, body) + _ = memInputs{inputs}.Keep(context.Background(), in.Digest, body) + rd.InputState = kvRead(inputs) + if all, _, err := rd.Inputs(""); err != nil || string(all[in.Digest]) != fmt.Sprintf(`{"size":%d}`, len(body)) { + t.Errorf("inputs = %s, %v", all, err) + } + if one, _, err := rd.Inputs(in.Digest); err != nil || string(one[in.Digest]) != string(body) { + t.Errorf("one input = %s, %v", one, err) + } + if _, _, err := rd.Inputs(strings.Repeat("0", 64)); err == nil { + t.Error("an input nothing kept was answered") + } } // 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head. diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index d3eb73ff..21b08c29 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -254,7 +254,8 @@ func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked { t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running) } - if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 { + // Two check asks and the checking program's two run asks (the seat list and the verdict). + if len(got.Steps) != 4 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 4 { t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks) } for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage) diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 1f71c257..c483b32a 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -20,6 +20,7 @@ import ( type state struct { Name string `json:"name"` History int `json:"history,omitempty"` + TTL int `json:"ttl-seconds,omitempty"` } type artifact struct { @@ -101,7 +102,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("module = %q", m.Module) } // delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets. - if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", @@ -163,7 +164,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). - if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"checker_ask_again", "checker_inputs", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code. diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 23dcc24a..b162fdeb 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -23,6 +23,7 @@ "state": [ { "name": "asks", "history": 1 }, { "name": "hearing", "history": 1 }, + { "name": "inputs", "ttl-seconds": 86400 }, { "name": "kept", "history": 1 }, { "name": "owed", "history": 5 }, { "name": "previews", "history": 5 }, @@ -52,6 +53,7 @@ ], "tools": [ "checker_ask_again", + "checker_inputs", "checker_kept", "checker_previews", "checker_pull", diff --git a/modules/mesh-delivery-checker/rules/program.go b/modules/mesh-delivery-checker/rules/program.go new file mode 100644 index 00000000..ca954634 --- /dev/null +++ b/modules/mesh-delivery-checker/rules/program.go @@ -0,0 +1,46 @@ +package rules + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" +) + +// Program is the checking program's work (cmd/mesh-checking-program; novox/hq ADR 0644, ADR 0656): over its arguments, +// its input (read only when asked for the verdict) and the heads laid for it (nil when none are), the one JSON line it +// answers, or why it refuses. `-seats` answers the seat list of the heads (SeatsOf), `-verdict` the verdict of the +// facts in the input (Judged), refusing facts of a version it does not read. +func Program(args []string, input func() ([]byte, error), heads fs.FS) ([]byte, error) { + set := flag.NewFlagSet("mesh-checking-program", flag.ContinueOnError) + set.SetOutput(io.Discard) + verdict := set.Bool("verdict", false, "answer the verdict of the facts in the input") + seats := set.Bool("seats", false, "answer the seat list of the heads laid for it") + if err := set.Parse(args); err != nil { + return nil, err + } + switch { + case *seats: + if heads == nil { + return nil, errors.New("no heads are laid for it") + } + answer, err := SeatsOf(heads) + if err != nil { + return nil, err + } + return json.Marshal(answer) + case *verdict: + data, err := input() + if err != nil { + return nil, fmt.Errorf("the input cannot be read: %w", err) + } + facts, err := DecodeFacts(data) + if err != nil { + return nil, err + } + return json.Marshal(Judged(facts)) + } + return nil, errors.New("nothing asked: give -verdict or -seats") +}