diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index d0bf7e9..cf57e05 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -237,8 +237,9 @@ func checkAskID(id string) error { } // StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job. -// Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's -// fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build. +// Exactly one of Build, Check, Compose and Run is set: a delivery ask is a build; a check ask a build (a candidate's +// fingerprint), a check or, on the Go seat, a compose or a run (RunJob); a recheck ask a check; an image ask is always a +// build. type StepAsk struct { V int `json:"v"` ID string `json:"id"` @@ -251,6 +252,7 @@ type StepAsk struct { Build *BuildJob `json:"build,omitempty"` Check *CheckJob `json:"check,omitempty"` Compose *ComposeJob `json:"compose,omitempty"` + Run *RunJob `json:"run,omitempty"` // Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node. Needs []string `json:"needs,omitempty"` // Packages are the version of each mesh package the build or the check builds against, by its module path (a Go @@ -409,16 +411,18 @@ func (a StepAsk) check() error { return fmt.Errorf("delivery: the ask %s names no time", a.ID) } jobs := 0 - for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} { + for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil, a.Run != nil} { if set { jobs++ } } switch { case jobs != 1: - return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID) + return fmt.Errorf("delivery: the ask %s holds a build, a check, a compose or a run, exactly one", a.ID) case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo): return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID) + case a.Run != nil && (a.Kind != AskCheck || a.Language != AskGo): + return fmt.Errorf("delivery: the ask %s is a run, which the Go seat's check stream alone carries", a.ID) case a.Build != nil && a.Kind == AskRecheck: return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) case a.Check != nil && a.Kind == AskDelivery: @@ -444,6 +448,9 @@ func (a StepAsk) check() error { if a.Compose != nil { return a.Compose.check(a.ID) } + if a.Run != nil { + return a.Run.check(a.ID) + } return a.Check.check(a.ID) } diff --git a/go/delivery/stepasks_compose_test.go b/go/delivery/stepasks_compose_test.go index 06ded5b..f368409 100644 --- a/go/delivery/stepasks_compose_test.go +++ b/go/delivery/stepasks_compose_test.go @@ -16,20 +16,36 @@ func composeAsk() StepAsk { Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}} } -// 1622: a compose ask encodes and decodes with every field. +// runAsk is the checker's run of a checking program built at a mesh-delivery head, over two heads (issue 1921, B0). +func runAsk() StepAsk { + in, err := RunInputOf("mesh-delivery-checker.inputs", []byte(`{"v":1}`)) + if err != nil { + panic(err) + } + return StepAsk{ID: "mh3k2x9qa1b2c6", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-checker", + At: time.Date(2026, 11, 6, 22, 0, 0, 0, time.UTC), + Run: &RunJob{Program: RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", + Package: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/cmd/mesh-checking-program"}, + Heads: []RunHead{{Repository: "novox/mesh-delivery", Commit: askCommit}, {Repository: "novox/mesh-sdk", Commit: askCommit}}, + Input: in, Args: []string{"-verdict"}, Reader: "checking program of mesh-delivery head"}, + Packages: PackageVersions{"git.novox.be/novox/mesh-sdk/go": askCommit}} +} + +// 1622: a compose ask encodes and decodes with every field; 1921 (B0): a run ask too. func TestAComposeAskRoundTrips(t *testing.T) { - a := composeAsk() - raw, err := EncodeStepAsk(a) - if err != nil { - t.Fatal(err) - } - got, err := DecodeStepAsk(raw) - if err != nil { - t.Fatal(err) - } - a.V = StepAskVersion - if !reflect.DeepEqual(got, a) { - t.Fatalf("round trip:\n got %+v\nwant %+v", got, a) + for _, a := range []StepAsk{composeAsk(), runAsk()} { + raw, err := EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeStepAsk(raw) + if err != nil { + t.Fatal(err) + } + a.V = StepAskVersion + if !reflect.DeepEqual(got, a) { + t.Fatalf("round trip:\n got %+v\nwant %+v", got, a) + } } } @@ -54,6 +70,41 @@ func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) { t.Errorf("a compose ask %s is encoded", name) } } + // 1921 (B0): a run ask likewise goes only on the Go seat's check stream, alone, builds a clean package at a full + // commit, reads each named head once at a full commit, and names an input kept by digest within its bound. + for name, change := range map[string]func(*StepAsk){ + "on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery }, + "on the Node seat": func(a *StepAsk) { a.Language = AskNode }, + "beside a compose": func(a *StepAsk) { a.Compose = composeAsk().Compose }, + "with a short commit": func(a *StepAsk) { a.Run.Program.Commit = "0123456" }, + "outside the repository": func(a *StepAsk) { a.Run.Program.Dir = "../x" }, + "with no package": func(a *StepAsk) { a.Run.Program.Package = "" }, + "with no head": func(a *StepAsk) { a.Run.Heads = nil }, + "with a head twice": func(a *StepAsk) { a.Run.Heads = append(a.Run.Heads, a.Run.Heads[0]) }, + "with a head at a branch": func(a *StepAsk) { a.Run.Heads[1].Commit = "main" }, + "with an input in no state": func(a *StepAsk) { a.Run.Input.State = "inputs" }, + "with an input by no digest": func(a *StepAsk) { a.Run.Input.Digest = "d1" }, + "with an input over its bound": func(a *StepAsk) { a.Run.Input.Size = RunInputBound + 1 }, + "with an argument of the shell": func(a *StepAsk) { a.Run.Args = []string{"-x; rm"} }, + } { + a := runAsk() + change(&a) + if _, err := EncodeStepAsk(a); err == nil { + t.Errorf("a run ask %s is encoded", name) + } + } + // The asker refuses an input over its bound before it asks, naming its size; the build module refuses bytes that + // are not the input named. + if _, err := RunInputOf("mesh-delivery-checker.inputs", make([]byte, RunInputBound+1)); err == nil || !strings.Contains(err.Error(), "786433 bytes") { + t.Errorf("an input over its bound: %v", err) + } + in := runAsk().Run.Input + if err := in.Verify([]byte(`{"v":1}`)); err != nil { + t.Errorf("the input named: %v", err) + } + if err := in.Verify([]byte(`{"v":2}`)); err == nil { + t.Error("another input read as the one named") + } } // 1622/AC3, 1921: a composer is kept under one key per repository, commit, directory, package, toolchain and asked versions, so two asks of @@ -86,4 +137,15 @@ func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) { if ComposerKey("r", askCommit, "go", "p", tc, nil) != ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{}) { t.Error("an empty ask changed the key") } + // 1921 (B0): a run's program is kept the same way, under a key of its own kind. + p := RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", Package: "p"} + rk := RunKey(p, tc, nil) + if !strings.HasPrefix(rk, "program-") || len(rk) != len("program-")+16 || rk != RunKey(p, tc, nil) { + t.Errorf("run key = %q", rk) + } + q := p + q.Commit = strings.Repeat("1", 40) + if RunKey(q, tc, nil) == rk || RunKey(p, tc, PackageVersions{"m/x": a}) == rk { + t.Error("two programs built from different inputs share the key") + } } diff --git a/go/delivery/stepasks_run.go b/go/delivery/stepasks_run.go new file mode 100644 index 0000000..8cbfb73 --- /dev/null +++ b/go/delivery/stepasks_run.go @@ -0,0 +1,162 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "path" + "regexp" + "strings" +) + +// **A run ask has a program built from a change and run in the build seat's sandbox** (novox/hq ADR 0644, ADR 0656; +// design 47, *What ADR 0656 leaves to this design*: the build seat's ask `run`; issue 1921). It goes on the Go seat's +// check stream, a whole job pulled where there is capacity, like a compose: the build module fetches Program at its +// commit and builds it once per RunKey, in a step that runs nothing but the build; fetches every head of Heads and lays +// each read-only under the directory RunHeadsEnv names, at its repository's path (`/`); reads the input +// Input names, checks its digest and writes it to the file RunInputEnv names; then runs the program with Args, no +// network and no bus account (ADR 0448 §2, §4), and answers RunAnswer on its step key. +// +// The input is carried by reference, as a compose carries its manifests: the asker keeps it in a state of its own by its +// digest (RunInput), measured against RunInputBound before it asks, so a message on the bus never carries it (issue 1930). + +// RunJob is a run ask's job. +type RunJob struct { + // Program is where the program is built: the repository and full commit fetched, the Go module's directory in it and + // the program's import path. The ask's Packages are the mesh package versions it builds against (ADR 0573 §2, §4). + Program RunProgram `json:"program"` + // Heads are the commits the program reads, each laid read-only at its repository's path; a repository at most once. + Heads []RunHead `json:"heads"` + // Input names the program's input, kept by the asker by its digest. + Input RunInput `json:"input"` + // Args are the program's arguments (a checking program's `-seats` or `-verdict`), each a word or a flag. + Args []string `json:"args,omitempty"` + // Reader names, for a reader of the ask, what the run is for ("checking program of mesh-delivery head"). + Reader string `json:"reader,omitempty"` +} + +// RunProgram is where a run's program is built from. +type RunProgram struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Dir string `json:"dir"` + Package string `json:"package"` +} + +// RunHead is one commit a run's program reads. +type RunHead struct { + Repository string `json:"repository"` + Commit string `json:"commit"` +} + +// RunInput is a run's input by reference: the state it is kept in, named as a manifest's `reads` names a state +// (`.`), its key (the sha256 of its bytes, 64 lowercase hex digits) and its size in bytes. +type RunInput struct { + State string `json:"state"` + Digest string `json:"digest"` + Size int `json:"size"` +} + +// RunInputBound is the most bytes a run's input may hold: under the bus's message limit, so the asker can keep it as +// one value. +const RunInputBound = 768 << 10 + +// RunHeadsEnv names, for the program, the directory its heads are laid in; RunInputEnv the file its input is written to. +const ( + RunHeadsEnv = "MESH_RUN_HEADS" + RunInputEnv = "MESH_RUN_INPUT" +) + +// RunInputOf is the reference to body kept in state, refusing a body over RunInputBound, naming its size, so an asker +// refuses it before asking. +func RunInputOf(state string, body []byte) (RunInput, error) { + if len(body) > RunInputBound { + return RunInput{}, fmt.Errorf("delivery: a run's input of %d bytes is over its bound of %d bytes", len(body), RunInputBound) + } + sum := sha256.Sum256(body) + in := RunInput{State: state, Digest: hex.EncodeToString(sum[:]), Size: len(body)} + return in, in.check("") +} + +// Verify refuses body when it is not the input i names: another size or another digest. +func (i RunInput) Verify(body []byte) error { + sum := sha256.Sum256(body) + if len(body) != i.Size || hex.EncodeToString(sum[:]) != i.Digest { + return fmt.Errorf("delivery: the input read from %s under %s is not the one the ask names (%d bytes, %d named)", i.State, i.Digest, len(body), i.Size) + } + return nil +} + +var ( + runStateOK = regexp.MustCompile(`^[a-z0-9][-a-z0-9]*\.[a-z0-9][-a-z0-9]*$`) + runDigestOK = regexp.MustCompile(`^[0-9a-f]{64}$`) + runArgOK = regexp.MustCompile(`^[-=_./a-zA-Z0-9]+$`) +) + +func (i RunInput) check(id string) error { + switch { + case !runStateOK.MatchString(i.State): + return fmt.Errorf("delivery: the run ask %s keeps its input in %q, which is no .", id, i.State) + case !runDigestOK.MatchString(i.Digest): + return fmt.Errorf("delivery: the run ask %s names its input by %q, which is no sha256 digest", id, i.Digest) + case i.Size <= 0 || i.Size > RunInputBound: + return fmt.Errorf("delivery: the run ask %s names an input of %d bytes, outside 1 to %d", id, i.Size, RunInputBound) + } + return nil +} + +func (r RunJob) check(id string) error { + p := r.Program + if err := checkRepository(p.Repository); err != nil { + return fmt.Errorf("the run ask %s's program: %w", id, err) + } + if !askCommitOK.MatchString(p.Commit) { + return fmt.Errorf("delivery: the run ask %s builds its program at no full commit", id) + } + if p.Dir == "" || p.Dir != path.Clean(p.Dir) || strings.HasPrefix(p.Dir, "/") || p.Dir == ".." || strings.HasPrefix(p.Dir, "../") { + return fmt.Errorf("delivery: the run ask %s names no clean directory inside the repository (%q)", id, p.Dir) + } + if p.Package == "" || strings.ContainsAny(p.Package, " \t\n") { + return fmt.Errorf("delivery: the run ask %s names no package to build (%q)", id, p.Package) + } + if len(r.Heads) == 0 { + return fmt.Errorf("delivery: the run ask %s names no head", id) + } + seen := map[string]bool{} + for _, h := range r.Heads { + if err := checkRepository(h.Repository); err != nil { + return fmt.Errorf("the run ask %s's head: %w", id, err) + } + if !askCommitOK.MatchString(h.Commit) { + return fmt.Errorf("delivery: the run ask %s names the head of %s at no full commit", id, h.Repository) + } + if seen[h.Repository] { + return fmt.Errorf("delivery: the run ask %s names %s twice", id, h.Repository) + } + seen[h.Repository] = true + } + for _, a := range r.Args { + if !runArgOK.MatchString(a) { + return fmt.Errorf("delivery: the run ask %s names the argument %q, which is no word or flag", id, a) + } + } + return r.Input.check(id) +} + +// RunAnswer is a run ask's step result: whether this run built the program or took the one kept under its key, the +// key, the program's exit, the last line of its standard output (Output, raw when it is JSON, else as a JSON string), +// and Said: the first line of its standard error, or why the step could not run it. +type RunAnswer struct { + Built bool `json:"built"` + Key string `json:"key"` + Exit int `json:"exit"` + Output json.RawMessage `json:"output,omitempty"` + Said string `json:"said,omitempty"` +} + +// RunKey is the key a build module keeps a built program under, as ComposerKey keeps a composer: what it was built +// from, the toolchain and the asked package versions, hashed (`program-` and 16 hex digits). +func RunKey(p RunProgram, toolchain ToolchainUsed, versions PackageVersions) string { + return "program-" + strings.TrimPrefix(ComposerKey(p.Repository, p.Commit, p.Dir, p.Package, toolchain, versions), "composer-") +}