Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b073c23c2 | ||
|
|
bdf44cd8e0 | ||
|
|
5cd8bf9520 | ||
|
|
336fd73b8b | ||
|
|
0364fcccc0 | ||
|
|
ceb319fb86 | ||
|
|
a0e5530aef | ||
|
|
0fbc4ce6f8 | ||
|
|
db4eff3885 |
@@ -118,6 +118,12 @@ func AskWhat(a delivery.StepAsk) string {
|
||||
}
|
||||
case a.Compose != nil:
|
||||
parts = append(parts, "compose", str(a.Compose.Fetch, "repository")+"@"+short(str(a.Compose.Fetch, "commit")))
|
||||
case a.Run != nil:
|
||||
parts = append(parts, "run "+a.Run.Program.Package, a.Run.Program.Repository+"@"+short(a.Run.Program.Commit))
|
||||
if len(a.Run.Args) > 0 {
|
||||
parts = append(parts, strings.Join(a.Run.Args, " "))
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%d heads", len(a.Run.Heads)))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
@@ -159,4 +159,9 @@ func TestAskWhat(t *testing.T) {
|
||||
if got, want := AskWhat(build), "delivery ask, module mesh-x, fp abcdef012345"; got != want {
|
||||
t.Errorf("build = %q", got)
|
||||
}
|
||||
// 1921: a run ask's step names the program, where it is built, its arguments and how many heads it reads.
|
||||
run := delivery.StepAsk{Kind: "check", Run: &delivery.RunJob{Program: delivery.RunProgram{Repository: "novox/mesh-delivery", Commit: "fedcba9876543210", Package: "./cmd/mesh-checking-program"}, Heads: []delivery.RunHead{{}, {}}, Args: []string{"-verdict"}}}
|
||||
if got, want := AskWhat(run), "check ask, run ./cmd/mesh-checking-program, novox/mesh-delivery@fedcba987654, -verdict, 2 heads"; got != want {
|
||||
t.Errorf("run = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ package catalogue
|
||||
// CheckerSeat is the seat mesh-delivery-checker holds.
|
||||
const CheckerSeat = "mesh-delivery-checker"
|
||||
|
||||
// checkerVerbs are the checker seat's verbs: ask a try, read one, end one by hand. Optional until the checker serves them
|
||||
// everywhere.
|
||||
// checkerVerbs are the checker seat's verbs: ask a try, read one, end one by hand, and answer a checking program's input.
|
||||
// Optional until the checker serves them everywhere.
|
||||
func checkerVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "try", Optional: true, Description: "Try a pushed head before its pull request is opened: the checker " +
|
||||
@@ -28,5 +28,12 @@ func checkerVerbs() []Verb {
|
||||
"id. A try that ended already is refused.",
|
||||
Input: schema(map[string]string{"id": "the try's id, try-<digits>", "why": "why it is ended, in words"},
|
||||
[]string{"id", "why"})},
|
||||
{Name: "input", Optional: true, Description: "A checking program's input the checker keeps, by the sha256 of its " +
|
||||
"exact bytes: the facts of a check its verdict is asked of, or the heads its seat list is asked of, as a run " +
|
||||
"ask names it (its input's verb and digest). Answers the digest and the bytes, base64; a digest it keeps no " +
|
||||
"input of is an error. Asked by the build module that runs the program, as its manifest's invokes grants it. " +
|
||||
"Read-only.",
|
||||
Input: schema(map[string]string{"digest": "the input's sha256, 64 lowercase hex digits, as the run ask names it"},
|
||||
[]string{"digest"})},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,11 @@ type Request struct {
|
||||
// longer read: the composer judges no manifest (novox/hq ADR 0578 §5), the indexer's judge does, and its verdicts
|
||||
// come in each case's Verdicts. Kept so an asker that still sends it is read.
|
||||
Predefined catalogue.PredefinedList `json:"predefined,omitempty"`
|
||||
// Seats are the seat definitions the manifest validator judges and the composer composes against, raw so that this
|
||||
// package names no seat: inside a group's check, the one seat list the checking program made from the members' heads
|
||||
// (novox/hq ADR 0636 §4, ADR 0644). A reader built before them ignores them; Decode, Fill and the request's
|
||||
// marshalling keep them whole, so a build module that fills a request hands them on (issue 1921).
|
||||
Seats []json.RawMessage `json:"seats,omitempty"`
|
||||
}
|
||||
|
||||
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
|
||||
|
||||
@@ -12,7 +12,9 @@ const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files
|
||||
// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its
|
||||
// place; a raw manifest goes as itself.
|
||||
func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
|
||||
r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
|
||||
// 1921 (B0): the seat list rides whole through the wire, the decode and the fill.
|
||||
seats := []json.RawMessage{json.RawMessage(`{"name":"node-files","verbs":["read"]}`)}
|
||||
r := Request{V: Version, Seats: seats, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
|
||||
"demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}},
|
||||
Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}}
|
||||
wire, err := json.Marshal(r)
|
||||
@@ -36,6 +38,10 @@ func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refilled, err := json.Marshal(back)
|
||||
if err != nil || !strings.Contains(string(refilled), `"seats":[{"name":"node-files","verbs":["read"]}]`) {
|
||||
t.Fatalf("the seats after the fill: %s %v", refilled, err)
|
||||
}
|
||||
if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo {
|
||||
t.Fatalf("filled %d times: %+v", asked, back.Cases[0])
|
||||
}
|
||||
|
||||
@@ -150,11 +150,6 @@ const (
|
||||
// WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder
|
||||
// writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published.
|
||||
WaitsFor = "waits for: "
|
||||
// Retake: a failure that passes by itself (a name lookup that timed out, a refused connection): the builder takes
|
||||
// the key again after a backoff, at most MaxRetakes times (issue 1748). A build module opens every such failure with it.
|
||||
Retake = "retakeable: "
|
||||
// MaxRetakes is how many times the builder takes a key again by itself for a retakeable failure.
|
||||
MaxRetakes = 3
|
||||
)
|
||||
|
||||
// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
|
||||
@@ -164,12 +159,6 @@ func FailedUntilChanged(err string) bool {
|
||||
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
|
||||
}
|
||||
|
||||
// Retakeable says whether a failed key's error opens with Retake: the builder takes it again by itself after a backoff,
|
||||
// until BuilderBuild.Retakes reaches MaxRetakes. An error that is FailedUntilChanged, or opens with no prefix, is not.
|
||||
func Retakeable(err string) bool {
|
||||
return strings.HasPrefix(err, Retake) && !FailedUntilChanged(err)
|
||||
}
|
||||
|
||||
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
|
||||
// the build module's recipe version.
|
||||
type ToolchainUsed struct {
|
||||
@@ -191,10 +180,6 @@ type BuilderBuild struct {
|
||||
Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
|
||||
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Retakes is how many times the builder took this key again by itself after a retakeable failure (Retake); it is kept
|
||||
// on the key, so a builder that starts again reads it and the count survives a restart (issue 1748). A reader that
|
||||
// does not know the field reads 0.
|
||||
Retakes int `json:"retakes,omitempty"`
|
||||
// Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry
|
||||
// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
|
||||
// clause 6). A reader that does not know the field reads the key as queued.
|
||||
|
||||
@@ -86,38 +86,11 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) {
|
||||
{VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false},
|
||||
{RefusedKind + "a package in go", false},
|
||||
{"go test: 2 failed", true},
|
||||
{Retake + "lookup git.novox.be: i/o timeout", true},
|
||||
} {
|
||||
if got := FailedUntilChanged(c.err); got == c.again {
|
||||
t.Errorf("FailedUntilChanged(%q) = %v", c.err, got)
|
||||
}
|
||||
}
|
||||
// A retakeable error opens with Retake and is not one that waits for the fingerprint; the count is on the key's value (issue 1748).
|
||||
for _, c := range []struct {
|
||||
err string
|
||||
want bool
|
||||
}{
|
||||
{Retake + "lookup git.novox.be: i/o timeout", true},
|
||||
{NotPinned + "x", false},
|
||||
{Retake[:4] + " not the prefix", false},
|
||||
{"go test: 2 failed", false},
|
||||
{"", false},
|
||||
{VersionTaken + Retake, false},
|
||||
} {
|
||||
if got := Retakeable(c.err); got != c.want {
|
||||
t.Errorf("Retakeable(%q) = %v", c.err, got)
|
||||
}
|
||||
}
|
||||
data, err := EncodeBuilderBuild(BuilderBuild{State: BuildFailed, Error: Retake + "x", Retakes: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back, err := DecodeBuilderBuild(data); err != nil || back.Retakes != 2 {
|
||||
t.Fatalf("decoded %+v, %v", back, err)
|
||||
}
|
||||
if Retake != "retakeable: " || MaxRetakes != 3 {
|
||||
t.Fatal("the retake prefix or its bound changed")
|
||||
}
|
||||
if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " {
|
||||
t.Fatal("a prefix changed its words")
|
||||
}
|
||||
|
||||
@@ -34,6 +34,10 @@ const (
|
||||
CheckerKeptHistory = 1
|
||||
)
|
||||
|
||||
// CheckerInputVerb is the verb of the seat mesh-delivery-checker that serves a checking program's input by its digest
|
||||
// (RunInput.Verb; asked RunInputAsk, answering RunInputAnswer; issue 1921, milestone 47/B0).
|
||||
const CheckerInputVerb = "input"
|
||||
|
||||
// The test levels (ADR 0445 clause 2) and the states of a verdict.
|
||||
const (
|
||||
LevelUnit = "unit"
|
||||
@@ -179,6 +183,40 @@ type Verdict struct {
|
||||
// carried over, by repository (ADR 0548 §5): the verdict was rewritten for Trunk and Beside by running only the packages
|
||||
// whose digest the moves changed. Absent from a verdict judged whole, and from one still running.
|
||||
CarriedFrom map[string]string `json:"carried-from,omitempty"`
|
||||
// JudgedBy names the checking program that judged the check: where it was built, the key its build is kept under and
|
||||
// the heads it read (novox/hq ADR 0644, ADR 0656; design 47's bootstrap delivery, *Its tool*). Absent in a value
|
||||
// written before it.
|
||||
JudgedBy *JudgedBy `json:"judged-by,omitempty"`
|
||||
}
|
||||
|
||||
// JudgedBy is the checking program's build that judged a check: built at Program, kept under Key (RunKey), Built when
|
||||
// this check's run built it rather than taking the kept one, over Heads.
|
||||
type JudgedBy struct {
|
||||
Program RunProgram `json:"program"`
|
||||
Key string `json:"key"`
|
||||
Built bool `json:"built,omitempty"`
|
||||
Heads []RunHead `json:"heads"`
|
||||
}
|
||||
|
||||
func (j *JudgedBy) check() error {
|
||||
if j == nil {
|
||||
return nil
|
||||
}
|
||||
if err := checkRepository(j.Program.Repository); err != nil || !askCommitOK.MatchString(j.Program.Commit) || j.Program.Package == "" {
|
||||
return fmt.Errorf("delivery: the verdict is judged by a checking program built at no repository, full commit and package (%+v)", j.Program)
|
||||
}
|
||||
if !strings.HasPrefix(j.Key, "program-") {
|
||||
return fmt.Errorf("delivery: the verdict's checking program is kept under %q, which is no program key", j.Key)
|
||||
}
|
||||
if len(j.Heads) == 0 {
|
||||
return fmt.Errorf("delivery: the verdict's checking program read no head")
|
||||
}
|
||||
for _, h := range j.Heads {
|
||||
if err := checkRepository(h.Repository); err != nil || !askCommitOK.MatchString(h.Commit) {
|
||||
return fmt.Errorf("delivery: the verdict's checking program read %s at %q, which is no full commit", h.Repository, h.Commit)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Composition) check() error {
|
||||
@@ -254,6 +292,9 @@ func (v Verdict) check() error {
|
||||
if err := v.Composition.check(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v.JudgedBy.check(); err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, level := range v.OverBound {
|
||||
if err := checkLevel(level); err != nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package delivery
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -314,6 +315,11 @@ func TestVerdictNewFieldsChecked(t *testing.T) {
|
||||
"carried from an empty commit": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": ""} }, []string{"carried-from", "mesh-sdk"}},
|
||||
"carried over no move": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": "aaaa"} }, []string{"nothing moved"}},
|
||||
"a running verdict carried": {func(v *Verdict) { v.State, v.CarriedFrom = VerdictRunning, map[string]string{"novox/mesh-sdk": "9999"} }, []string{"running", "carried"}},
|
||||
// 1921 (B0): the checking program that judged it.
|
||||
"judged by no program": {func(v *Verdict) { v.JudgedBy = &JudgedBy{Key: "program-0123456789abcdef", Heads: judgedBy().Heads} }, []string{"checking program", "full commit"}},
|
||||
"judged under no key": {func(v *Verdict) { j := judgedBy(); j.Key = "composer-1"; v.JudgedBy = &j }, []string{"composer-1"}},
|
||||
"judged over no head": {func(v *Verdict) { j := judgedBy(); j.Heads = nil; v.JudgedBy = &j }, []string{"no head"}},
|
||||
"judged over a branch": {func(v *Verdict) { j := judgedBy(); j.Heads[0].Commit = "main"; v.JudgedBy = &j }, []string{"novox/mesh-delivery", "main"}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
v := passVerdict()
|
||||
@@ -338,7 +344,7 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("the new reader of an old value: %v", err)
|
||||
}
|
||||
if v.Composition != nil || v.OverBound != nil || v.Beside != nil || v.CarriedFrom != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 {
|
||||
if v.Composition != nil || v.OverBound != nil || v.Beside != nil || v.CarriedFrom != nil || v.JudgedBy != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 {
|
||||
t.Errorf("an old value grew fields: %+v", v)
|
||||
}
|
||||
again, err := EncodeVerdict(v)
|
||||
@@ -352,10 +358,16 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
|
||||
n.Beside = map[string]string{"novox/mesh-catalog": "cccc"}
|
||||
n.CarriedFrom = map[string]string{"novox/mesh-sdk": "9999"}
|
||||
n.Composition.FPSet = "fps1:00"
|
||||
j := judgedBy()
|
||||
n.JudgedBy = &j
|
||||
data, err := EncodeVerdict(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 1921 (B0): who judged it reads back whole.
|
||||
if back, err := DecodeVerdict(data); err != nil || back.JudgedBy == nil || !reflect.DeepEqual(*back.JudgedBy, j) {
|
||||
t.Errorf("judged by, read back: %+v %v", back.JudgedBy, err)
|
||||
}
|
||||
var ov oldVerdict
|
||||
if err := json.Unmarshal(data, &ov); err != nil || ov.Group != "feat/x" || ov.State != VerdictPass || ov.Levels[LevelUnit].Run != 12 || ov.SDK == "" {
|
||||
t.Errorf("the old reader of a new value: %+v, %v", ov, err)
|
||||
@@ -658,3 +670,8 @@ func TestVerdictNamesEveryFailure(t *testing.T) {
|
||||
t.Error("a verdict naming more than MaxFailures and the rest was written")
|
||||
}
|
||||
}
|
||||
|
||||
func judgedBy() JudgedBy {
|
||||
return JudgedBy{Program: RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", Package: "p"},
|
||||
Key: "program-0123456789abcdef", Built: true, Heads: []RunHead{{Repository: "novox/mesh-delivery", Commit: askCommit}}}
|
||||
}
|
||||
|
||||
@@ -175,9 +175,6 @@ type BuildLogEntry struct {
|
||||
// sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer
|
||||
// keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's.
|
||||
Imports []string `json:"imports,omitempty"`
|
||||
// Retakes is, at a build's end, how many times the builder took the key again by itself after a retakeable failure
|
||||
// (BuilderBuild.Retakes; issue 1748): builder_builds_since reads it.
|
||||
Retakes int `json:"retakes,omitempty"`
|
||||
}
|
||||
|
||||
// BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held
|
||||
|
||||
+24
-7
@@ -237,8 +237,9 @@ func checkAskID(id string) error {
|
||||
}
|
||||
|
||||
// StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job.
|
||||
// Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's
|
||||
// fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build.
|
||||
// Exactly one of Build, Check, Compose and Run is set: a delivery ask is a build; a check ask a build (a candidate's
|
||||
// fingerprint), a check or, on the Go seat, a compose or a run (RunJob); a recheck ask a check; an image ask is always a
|
||||
// build.
|
||||
type StepAsk struct {
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
@@ -251,6 +252,7 @@ type StepAsk struct {
|
||||
Build *BuildJob `json:"build,omitempty"`
|
||||
Check *CheckJob `json:"check,omitempty"`
|
||||
Compose *ComposeJob `json:"compose,omitempty"`
|
||||
Run *RunJob `json:"run,omitempty"`
|
||||
// Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node.
|
||||
Needs []string `json:"needs,omitempty"`
|
||||
// Packages are the version of each mesh package the build or the check builds against, by its module path (a Go
|
||||
@@ -310,9 +312,19 @@ type ComposeAnswer struct {
|
||||
|
||||
// ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it
|
||||
// was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once
|
||||
// on a node (ADR 0482 clause 4).
|
||||
func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed) string {
|
||||
sum := sha256.Sum256([]byte(strings.Join([]string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}, "\n")))
|
||||
// on a node (ADR 0482 clause 4). The mesh package versions the ask names are an input of the build, so they are in the
|
||||
// key, sorted, when there are any; an ask that names none gets the key it always had (issue 1921).
|
||||
func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed, versions PackageVersions) string {
|
||||
parts := []string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}
|
||||
pkgs := make([]string, 0, len(versions))
|
||||
for p := range versions {
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
sort.Strings(pkgs)
|
||||
for _, p := range pkgs {
|
||||
parts = append(parts, p+"="+versions[p])
|
||||
}
|
||||
sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
|
||||
return "composer-" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
@@ -399,16 +411,18 @@ func (a StepAsk) check() error {
|
||||
return fmt.Errorf("delivery: the ask %s names no time", a.ID)
|
||||
}
|
||||
jobs := 0
|
||||
for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} {
|
||||
for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil, a.Run != nil} {
|
||||
if set {
|
||||
jobs++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case jobs != 1:
|
||||
return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID)
|
||||
return fmt.Errorf("delivery: the ask %s holds a build, a check, a compose or a run, exactly one", a.ID)
|
||||
case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo):
|
||||
return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID)
|
||||
case a.Run != nil && (a.Kind != AskCheck || a.Language != AskGo):
|
||||
return fmt.Errorf("delivery: the ask %s is a run, which the Go seat's check stream alone carries", a.ID)
|
||||
case a.Build != nil && a.Kind == AskRecheck:
|
||||
return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID)
|
||||
case a.Check != nil && a.Kind == AskDelivery:
|
||||
@@ -434,6 +448,9 @@ func (a StepAsk) check() error {
|
||||
if a.Compose != nil {
|
||||
return a.Compose.check(a.ID)
|
||||
}
|
||||
if a.Run != nil {
|
||||
return a.Run.check(a.ID)
|
||||
}
|
||||
return a.Check.check(a.ID)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -16,20 +17,36 @@ func composeAsk() StepAsk {
|
||||
Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}}
|
||||
}
|
||||
|
||||
// 1622: a compose ask encodes and decodes with every field.
|
||||
// runAsk is the checker's run of a checking program built at a mesh-delivery head, over two heads (issue 1921, B0).
|
||||
func runAsk() StepAsk {
|
||||
in, err := RunInputOf("seat:mesh-delivery-checker."+CheckerInputVerb, []byte(`{"v":1}`))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return StepAsk{ID: "mh3k2x9qa1b2c6", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-checker",
|
||||
At: time.Date(2026, 11, 6, 22, 0, 0, 0, time.UTC),
|
||||
Run: &RunJob{Program: RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker",
|
||||
Package: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/cmd/mesh-checking-program"},
|
||||
Heads: []RunHead{{Repository: "novox/mesh-delivery", Commit: askCommit}, {Repository: "novox/mesh-sdk", Commit: askCommit}},
|
||||
Input: in, Args: []string{"-verdict"}, Reader: "checking program of mesh-delivery head"},
|
||||
Packages: PackageVersions{"git.novox.be/novox/mesh-sdk/go": askCommit}}
|
||||
}
|
||||
|
||||
// 1622: a compose ask encodes and decodes with every field; 1921 (B0): a run ask too.
|
||||
func TestAComposeAskRoundTrips(t *testing.T) {
|
||||
a := composeAsk()
|
||||
raw, err := EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := DecodeStepAsk(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.V = StepAskVersion
|
||||
if !reflect.DeepEqual(got, a) {
|
||||
t.Fatalf("round trip:\n got %+v\nwant %+v", got, a)
|
||||
for _, a := range []StepAsk{composeAsk(), runAsk()} {
|
||||
raw, err := EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := DecodeStepAsk(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.V = StepAskVersion
|
||||
if !reflect.DeepEqual(got, a) {
|
||||
t.Fatalf("round trip:\n got %+v\nwant %+v", got, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,23 +71,112 @@ func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) {
|
||||
t.Errorf("a compose ask %s is encoded", name)
|
||||
}
|
||||
}
|
||||
// 1921 (B0): a run ask likewise goes only on the Go seat's check stream, alone, builds a clean package at a full
|
||||
// commit, reads each named head once at a full commit, and names an input kept by digest within its bound.
|
||||
for name, change := range map[string]func(*StepAsk){
|
||||
"on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery },
|
||||
"on the Node seat": func(a *StepAsk) { a.Language = AskNode },
|
||||
"beside a compose": func(a *StepAsk) { a.Compose = composeAsk().Compose },
|
||||
"with a short commit": func(a *StepAsk) { a.Run.Program.Commit = "0123456" },
|
||||
"outside the repository": func(a *StepAsk) { a.Run.Program.Dir = "../x" },
|
||||
"with no package": func(a *StepAsk) { a.Run.Program.Package = "" },
|
||||
"with no head": func(a *StepAsk) { a.Run.Heads = nil },
|
||||
"with a head twice": func(a *StepAsk) { a.Run.Heads = append(a.Run.Heads, a.Run.Heads[0]) },
|
||||
"with a head at a branch": func(a *StepAsk) { a.Run.Heads[1].Commit = "main" },
|
||||
"with an input on a state": func(a *StepAsk) { a.Run.Input.Verb = "mesh-delivery-checker.inputs" },
|
||||
"with an input on one machine": func(a *StepAsk) { a.Run.Input.Verb = "seat:mesh-delivery-checker.input@ace" },
|
||||
"with an input by no digest": func(a *StepAsk) { a.Run.Input.Digest = "d1" },
|
||||
"with an input over its bound": func(a *StepAsk) { a.Run.Input.Size = RunInputBound + 1 },
|
||||
"with an argument of the shell": func(a *StepAsk) { a.Run.Args = []string{"-x; rm"} },
|
||||
} {
|
||||
a := runAsk()
|
||||
change(&a)
|
||||
if _, err := EncodeStepAsk(a); err == nil {
|
||||
t.Errorf("a run ask %s is encoded", name)
|
||||
}
|
||||
}
|
||||
// The asker refuses an input over its bound before it asks, naming its size; the build module refuses bytes that
|
||||
// are not the input named.
|
||||
if _, err := RunInputOf("seat:mesh-delivery-checker.input", make([]byte, RunInputBound+1)); err == nil || !strings.Contains(err.Error(), "262145 bytes") {
|
||||
t.Errorf("an input over its bound: %v", err)
|
||||
}
|
||||
in := runAsk().Run.Input
|
||||
if err := in.Verify([]byte(`{"v":1}`)); err != nil {
|
||||
t.Errorf("the input named: %v", err)
|
||||
}
|
||||
if err := in.Verify([]byte(`{"v":2}`)); err == nil {
|
||||
t.Error("another input read as the one named")
|
||||
}
|
||||
// The build module asks the verb the input names for its digest and takes only the bytes named (the operator's answer
|
||||
// on B0: the input travels by a verb of the asker's seat, never a read of its state).
|
||||
for name, c := range map[string]struct {
|
||||
answer string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
"the bytes named": {answer: `{"digest":"` + in.Digest + `","input":"eyJ2IjoxfQ=="}`},
|
||||
"other bytes": {answer: `{"digest":"` + in.Digest + `","input":"eyJ2IjoyfQ=="}`, want: "is not the one the ask names"},
|
||||
"another digest": {answer: `{"digest":"` + strings.Repeat("0", 64) + `","input":"eyJ2IjoxfQ=="}`, want: "answered the input"},
|
||||
"no input": {answer: `[1]`, want: "answered no input"},
|
||||
"an error of the callee": {err: errors.New("keeps no input of that digest"), want: "keeps no input of that digest"},
|
||||
} {
|
||||
var asked string
|
||||
var body any
|
||||
got, err := in.Read(func(key string, b any) (json.RawMessage, error) {
|
||||
asked, body = key, b
|
||||
return json.RawMessage(c.answer), c.err
|
||||
})
|
||||
if asked != "seat:mesh-delivery-checker.input" || body != (RunInputAsk{Digest: in.Digest}) {
|
||||
t.Errorf("%s: asked %s with %+v", name, asked, body)
|
||||
}
|
||||
switch {
|
||||
case c.want == "" && (err != nil || string(got) != `{"v":1}`):
|
||||
t.Errorf("%s: %q, %v", name, got, err)
|
||||
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want)):
|
||||
t.Errorf("%s: %v, want %q", name, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC3: a composer is kept under one key per repository, commit, directory, package and toolchain, so two asks of
|
||||
// 1622/AC3, 1921: a composer is kept under one key per repository, commit, directory, package, toolchain and asked versions, so two asks of
|
||||
// one head build it once on a node, and another head or toolchain builds its own.
|
||||
func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) {
|
||||
tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"}
|
||||
k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc)
|
||||
if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 {
|
||||
k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, nil)
|
||||
if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, nil) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 {
|
||||
t.Fatalf("key = %q", k)
|
||||
}
|
||||
for _, other := range []string{
|
||||
ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc),
|
||||
ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc),
|
||||
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}),
|
||||
ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc, nil),
|
||||
ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc, nil),
|
||||
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}, nil),
|
||||
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, PackageVersions{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("2", 40)}),
|
||||
} {
|
||||
if other == k {
|
||||
t.Errorf("two composers built from different inputs share the key %s", k)
|
||||
}
|
||||
}
|
||||
// 1921: the asked versions are in the key, in any order of naming; none, or an empty ask, gives the key without them.
|
||||
a, b := strings.Repeat("a", 40), strings.Repeat("b", 40)
|
||||
k1 := ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/x": a, "m/y": b})
|
||||
if k1 != ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/y": b, "m/x": a}) {
|
||||
t.Error("the key depends on the order of the versions")
|
||||
}
|
||||
if k1 == ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/x": b, "m/y": a}) || k1 == ComposerKey("r", askCommit, "go", "p", tc, nil) {
|
||||
t.Error("other versions, or none, share the key")
|
||||
}
|
||||
if ComposerKey("r", askCommit, "go", "p", tc, nil) != ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{}) {
|
||||
t.Error("an empty ask changed the key")
|
||||
}
|
||||
// 1921 (B0): a run's program is kept the same way, under a key of its own kind.
|
||||
p := RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", Package: "p"}
|
||||
rk := RunKey(p, tc, nil)
|
||||
if !strings.HasPrefix(rk, "program-") || len(rk) != len("program-")+16 || rk != RunKey(p, tc, nil) {
|
||||
t.Errorf("run key = %q", rk)
|
||||
}
|
||||
q := p
|
||||
q.Commit = strings.Repeat("1", 40)
|
||||
if RunKey(q, tc, nil) == rk || RunKey(p, tc, PackageVersions{"m/x": a}) == rk {
|
||||
t.Error("two programs built from different inputs share the key")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/calladdress"
|
||||
)
|
||||
|
||||
// **A run ask has a program built from a change and run in the build seat's sandbox** (novox/hq ADR 0644, ADR 0656;
|
||||
// design 47, *What ADR 0656 leaves to this design*: the build seat's ask `run`; issue 1921). It goes on the Go seat's
|
||||
// check stream, a whole job pulled where there is capacity, like a compose: the build module fetches Program at its
|
||||
// commit and builds it once per RunKey, in a step that runs nothing but the build; fetches every head of Heads and lays
|
||||
// each read-only under the directory RunHeadsEnv names, at its repository's path (`<owner>/<name>`); reads the input
|
||||
// Input names, checks its digest and writes it to the file RunInputEnv names; then runs the program with Args, no
|
||||
// network and no bus account (ADR 0448 §2, §4), and answers RunAnswer on its step key.
|
||||
//
|
||||
// The input is carried by reference, as a compose carries its manifests: the asker keeps it by its digest, measured
|
||||
// against RunInputBound before it asks, and serves it on a verb of its seat (RunInput.Verb, answered RunInputAnswer); the
|
||||
// build module asks that verb as its manifest's `invokes` grants it, never reading the asker's state (ADR 0596 §2; the
|
||||
// operator's answer on milestone 47/B0), so the ask on the stream never carries it (issue 1930).
|
||||
|
||||
// RunJob is a run ask's job.
|
||||
type RunJob struct {
|
||||
// Program is where the program is built: the repository and full commit fetched, the Go module's directory in it and
|
||||
// the program's import path. The ask's Packages are the mesh package versions it builds against (ADR 0573 §2, §4).
|
||||
Program RunProgram `json:"program"`
|
||||
// Heads are the commits the program reads, each laid read-only at its repository's path; a repository at most once.
|
||||
Heads []RunHead `json:"heads"`
|
||||
// Input names the program's input, kept by the asker by its digest and served on a verb of its seat.
|
||||
Input RunInput `json:"input"`
|
||||
// Args are the program's arguments (a checking program's `-seats` or `-verdict`), each a word or a flag.
|
||||
Args []string `json:"args,omitempty"`
|
||||
// Reader names, for a reader of the ask, what the run is for ("checking program of mesh-delivery head").
|
||||
Reader string `json:"reader,omitempty"`
|
||||
}
|
||||
|
||||
// RunProgram is where a run's program is built from.
|
||||
type RunProgram struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
}
|
||||
|
||||
// RunHead is one commit a run's program reads.
|
||||
type RunHead struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
// RunInput is a run's input by reference: the seat verb the asker serves it on, named as a manifest's `invokes` names it
|
||||
// (`seat:<seat>.<verb>`, asked of no one machine), its digest (the sha256 of its bytes, 64 lowercase hex digits) and its
|
||||
// size in bytes. The verb is asked RunInputAsk and answers RunInputAnswer.
|
||||
type RunInput struct {
|
||||
Verb string `json:"verb"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
|
||||
// RunInputAsk is what a verb a RunInput names is asked: the input's digest.
|
||||
type RunInputAsk struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// RunInputAnswer is that verb's answer: the digest asked and the input's exact bytes (base64 in JSON, so no encoding on
|
||||
// the way changes a byte the digest covers).
|
||||
type RunInputAnswer struct {
|
||||
Digest string `json:"digest"`
|
||||
Input []byte `json:"input"`
|
||||
}
|
||||
|
||||
// RunInputBound is the most bytes a run's input may hold: the bus's value limit the design already uses (256 KiB), so
|
||||
// the asker keeps it as one value (the lead's ruling on milestone 47/B0, 2026-11-06).
|
||||
const RunInputBound = 256 << 10
|
||||
|
||||
// RunHeadsEnv names, for the program, the directory its heads are laid in; RunInputEnv the file its input is written to.
|
||||
const (
|
||||
RunHeadsEnv = "MESH_RUN_HEADS"
|
||||
RunInputEnv = "MESH_RUN_INPUT"
|
||||
)
|
||||
|
||||
// RunInputOf is the reference to body served on verb (`seat:<seat>.<verb>`), refusing a body over RunInputBound, naming
|
||||
// its size, so an asker refuses it before asking.
|
||||
func RunInputOf(verb string, body []byte) (RunInput, error) {
|
||||
if len(body) > RunInputBound {
|
||||
return RunInput{}, fmt.Errorf("delivery: a run's input of %d bytes is over its bound of %d bytes", len(body), RunInputBound)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
in := RunInput{Verb: verb, Digest: hex.EncodeToString(sum[:]), Size: len(body)}
|
||||
return in, in.check("")
|
||||
}
|
||||
|
||||
// Verify refuses body when it is not the input i names: another size or another digest.
|
||||
func (i RunInput) Verify(body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
if len(body) != i.Size || hex.EncodeToString(sum[:]) != i.Digest {
|
||||
return fmt.Errorf("delivery: the input %s answered for %s is not the one the ask names (%d bytes, %d named)", i.Verb, i.Digest, len(body), i.Size)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read asks the verb i names for its input with ask (a tools bundle's stdio.Ask, a process's Conn.Call) and answers its
|
||||
// bytes, refusing an answer for another digest or bytes that are not the input named.
|
||||
func (i RunInput) Read(ask func(key string, body any) (json.RawMessage, error)) ([]byte, error) {
|
||||
raw, err := ask(i.Verb, RunInputAsk{Digest: i.Digest})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delivery: asking %s for the input %s: %w", i.Verb, i.Digest, err)
|
||||
}
|
||||
var a RunInputAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("delivery: %s answered no input for %s: %w", i.Verb, i.Digest, err)
|
||||
}
|
||||
if a.Digest != i.Digest {
|
||||
return nil, fmt.Errorf("delivery: %s answered the input %q, asked %s", i.Verb, a.Digest, i.Digest)
|
||||
}
|
||||
return a.Input, i.Verify(a.Input)
|
||||
}
|
||||
|
||||
var (
|
||||
runDigestOK = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
runArgOK = regexp.MustCompile(`^[-=_./a-zA-Z0-9]+$`)
|
||||
)
|
||||
|
||||
func (i RunInput) check(id string) error {
|
||||
switch {
|
||||
case !runVerbOK(i.Verb):
|
||||
return fmt.Errorf("delivery: the run ask %s serves its input on %q, which is no seat:<seat>.<verb> of no one machine", id, i.Verb)
|
||||
case !runDigestOK.MatchString(i.Digest):
|
||||
return fmt.Errorf("delivery: the run ask %s names its input by %q, which is no sha256 digest", id, i.Digest)
|
||||
case i.Size <= 0 || i.Size > RunInputBound:
|
||||
return fmt.Errorf("delivery: the run ask %s names an input of %d bytes, outside 1 to %d", id, i.Size, RunInputBound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runVerbOK says key names a seat's verb, asked of no one machine.
|
||||
func runVerbOK(key string) bool {
|
||||
a, err := calladdress.Parse(key)
|
||||
return err == nil && a.Seat != "" && a.Node == ""
|
||||
}
|
||||
|
||||
func (r RunJob) check(id string) error {
|
||||
p := r.Program
|
||||
if err := checkRepository(p.Repository); err != nil {
|
||||
return fmt.Errorf("the run ask %s's program: %w", id, err)
|
||||
}
|
||||
if !askCommitOK.MatchString(p.Commit) {
|
||||
return fmt.Errorf("delivery: the run ask %s builds its program at no full commit", id)
|
||||
}
|
||||
if p.Dir == "" || p.Dir != path.Clean(p.Dir) || strings.HasPrefix(p.Dir, "/") || p.Dir == ".." || strings.HasPrefix(p.Dir, "../") {
|
||||
return fmt.Errorf("delivery: the run ask %s names no clean directory inside the repository (%q)", id, p.Dir)
|
||||
}
|
||||
if p.Package == "" || strings.ContainsAny(p.Package, " \t\n") {
|
||||
return fmt.Errorf("delivery: the run ask %s names no package to build (%q)", id, p.Package)
|
||||
}
|
||||
if len(r.Heads) == 0 {
|
||||
return fmt.Errorf("delivery: the run ask %s names no head", id)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, h := range r.Heads {
|
||||
if err := checkRepository(h.Repository); err != nil {
|
||||
return fmt.Errorf("the run ask %s's head: %w", id, err)
|
||||
}
|
||||
if !askCommitOK.MatchString(h.Commit) {
|
||||
return fmt.Errorf("delivery: the run ask %s names the head of %s at no full commit", id, h.Repository)
|
||||
}
|
||||
if seen[h.Repository] {
|
||||
return fmt.Errorf("delivery: the run ask %s names %s twice", id, h.Repository)
|
||||
}
|
||||
seen[h.Repository] = true
|
||||
}
|
||||
for _, a := range r.Args {
|
||||
if !runArgOK.MatchString(a) {
|
||||
return fmt.Errorf("delivery: the run ask %s names the argument %q, which is no word or flag", id, a)
|
||||
}
|
||||
}
|
||||
return r.Input.check(id)
|
||||
}
|
||||
|
||||
// RunAnswer is a run ask's step result: whether this run built the program or took the one kept under its key, the
|
||||
// key, the program's exit, the last line of its standard output (Output, raw when it is JSON, else as a JSON string),
|
||||
// and Said: the first line of its standard error, or why the step could not run it.
|
||||
type RunAnswer struct {
|
||||
Built bool `json:"built"`
|
||||
Key string `json:"key"`
|
||||
Exit int `json:"exit"`
|
||||
Output json.RawMessage `json:"output,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// RunKey is the key a build module keeps a built program under, as ComposerKey keeps a composer: what it was built
|
||||
// from, the toolchain and the asked package versions, hashed (`program-` and 16 hex digits).
|
||||
func RunKey(p RunProgram, toolchain ToolchainUsed, versions PackageVersions) string {
|
||||
return "program-" + strings.TrimPrefix(ComposerKey(p.Repository, p.Commit, p.Dir, p.Package, toolchain, versions), "composer-")
|
||||
}
|
||||
Reference in New Issue
Block a user