Compare commits

..
Author SHA1 Message Date
jschoubben 0d97c09c49 Fold the retake test into the prefix test (issue 1748 review 1)
mesh/delivery-group group fix/1748-retake-and-resolver ready: every member ready, and composed together they pass
mesh/delivery closed: the pull request closed unmerged
mesh/check pass 0d97c09c49b4 on 5a38a1221659 | unit 6409 run, 187 kept, 14 skipped | integration 6544 run, 220 kept, 23 skipped | over bound: unit, in…
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:41:49 +01:00
jschoubben 9c4b2c5e56 A build's log entry says how many times its key was retaken (issue 1748)
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:27:52 +01:00
jschoubben 0261881feb Add the retake class beside FailedUntilChanged and the retake count on a build key (issue 1748)
A network failure passes by itself; the build contract names it so every build module is held to it and the builder acts on it.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:27:00 +01:00
13 changed files with 75 additions and 441 deletions
-6
View File
@@ -118,12 +118,6 @@ func AskWhat(a delivery.StepAsk) string {
} }
case a.Compose != nil: case a.Compose != nil:
parts = append(parts, "compose", str(a.Compose.Fetch, "repository")+"@"+short(str(a.Compose.Fetch, "commit"))) parts = append(parts, "compose", str(a.Compose.Fetch, "repository")+"@"+short(str(a.Compose.Fetch, "commit")))
case a.Run != nil:
parts = append(parts, "run "+a.Run.Program.Package, a.Run.Program.Repository+"@"+short(a.Run.Program.Commit))
if len(a.Run.Args) > 0 {
parts = append(parts, strings.Join(a.Run.Args, " "))
}
parts = append(parts, fmt.Sprintf("%d heads", len(a.Run.Heads)))
} }
return strings.Join(parts, ", ") return strings.Join(parts, ", ")
} }
-5
View File
@@ -159,9 +159,4 @@ func TestAskWhat(t *testing.T) {
if got, want := AskWhat(build), "delivery ask, module mesh-x, fp abcdef012345"; got != want { if got, want := AskWhat(build), "delivery ask, module mesh-x, fp abcdef012345"; got != want {
t.Errorf("build = %q", got) t.Errorf("build = %q", got)
} }
// 1921: a run ask's step names the program, where it is built, its arguments and how many heads it reads.
run := delivery.StepAsk{Kind: "check", Run: &delivery.RunJob{Program: delivery.RunProgram{Repository: "novox/mesh-delivery", Commit: "fedcba9876543210", Package: "./cmd/mesh-checking-program"}, Heads: []delivery.RunHead{{}, {}}, Args: []string{"-verdict"}}}
if got, want := AskWhat(run), "check ask, run ./cmd/mesh-checking-program, novox/mesh-delivery@fedcba987654, -verdict, 2 heads"; got != want {
t.Errorf("run = %q", got)
}
} }
+2 -9
View File
@@ -3,8 +3,8 @@ package catalogue
// CheckerSeat is the seat mesh-delivery-checker holds. // CheckerSeat is the seat mesh-delivery-checker holds.
const CheckerSeat = "mesh-delivery-checker" const CheckerSeat = "mesh-delivery-checker"
// checkerVerbs are the checker seat's verbs: ask a try, read one, end one by hand, and answer a checking program's input. // checkerVerbs are the checker seat's verbs: ask a try, read one, end one by hand. Optional until the checker serves them
// Optional until the checker serves them everywhere. // everywhere.
func checkerVerbs() []Verb { func checkerVerbs() []Verb {
return []Verb{ return []Verb{
{Name: "try", Optional: true, Description: "Try a pushed head before its pull request is opened: the checker " + {Name: "try", Optional: true, Description: "Try a pushed head before its pull request is opened: the checker " +
@@ -28,12 +28,5 @@ func checkerVerbs() []Verb {
"id. A try that ended already is refused.", "id. A try that ended already is refused.",
Input: schema(map[string]string{"id": "the try's id, try-<digits>", "why": "why it is ended, in words"}, Input: schema(map[string]string{"id": "the try's id, try-<digits>", "why": "why it is ended, in words"},
[]string{"id", "why"})}, []string{"id", "why"})},
{Name: "input", Optional: true, Description: "A checking program's input the checker keeps, by the sha256 of its " +
"exact bytes: the facts of a check its verdict is asked of, or the heads its seat list is asked of, as a run " +
"ask names it (its input's verb and digest). Answers the digest and the bytes, base64; a digest it keeps no " +
"input of is an error. Asked by the build module that runs the program, as its manifest's invokes grants it. " +
"Read-only.",
Input: schema(map[string]string{"digest": "the input's sha256, 64 lowercase hex digits, as the run ask names it"},
[]string{"digest"})},
} }
} }
-5
View File
@@ -59,11 +59,6 @@ type Request struct {
// longer read: the composer judges no manifest (novox/hq ADR 0578 §5), the indexer's judge does, and its verdicts // longer read: the composer judges no manifest (novox/hq ADR 0578 §5), the indexer's judge does, and its verdicts
// come in each case's Verdicts. Kept so an asker that still sends it is read. // come in each case's Verdicts. Kept so an asker that still sends it is read.
Predefined catalogue.PredefinedList `json:"predefined,omitempty"` Predefined catalogue.PredefinedList `json:"predefined,omitempty"`
// Seats are the seat definitions the manifest validator judges and the composer composes against, raw so that this
// package names no seat: inside a group's check, the one seat list the checking program made from the members' heads
// (novox/hq ADR 0636 §4, ADR 0644). A reader built before them ignores them; Decode, Fill and the request's
// marshalling keep them whole, so a build module that fills a request hands them on (issue 1921).
Seats []json.RawMessage `json:"seats,omitempty"`
} }
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs. // Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
+1 -7
View File
@@ -12,9 +12,7 @@ const demo = `{"module": "demo", "version": "1", "claims": [{"name": "node-files
// A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its // A manifest named by its digest goes on the wire as {"digest": ...}, and Fill puts the bytes the indexer keeps in its
// place; a raw manifest goes as itself. // place; a raw manifest goes as itself.
func TestAManifestIsItsBytesOrItsDigest(t *testing.T) { func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
// 1921 (B0): the seat list rides whole through the wire, the decode and the fill. r := Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
seats := []json.RawMessage{json.RawMessage(`{"name":"node-files","verbs":["read"]}`)}
r := Request{V: Version, Seats: seats, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{
"demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}}, "demo": {Digest: "d1"}, "raw": {Raw: json.RawMessage(`{"module":"raw","version":"1"}`)}},
Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}} Kept: map[string]map[string]Manifest{"laptop": {"demo": {Digest: "d1"}}}}}}
wire, err := json.Marshal(r) wire, err := json.Marshal(r)
@@ -38,10 +36,6 @@ func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
refilled, err := json.Marshal(back)
if err != nil || !strings.Contains(string(refilled), `"seats":[{"name":"node-files","verbs":["read"]}]`) {
t.Fatalf("the seats after the fill: %s %v", refilled, err)
}
if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo { if asked != 2 || string(back.Cases[0].Manifests["demo"].Raw) != demo || string(back.Cases[0].Kept["laptop"]["demo"].Raw) != demo {
t.Fatalf("filled %d times: %+v", asked, back.Cases[0]) t.Fatalf("filled %d times: %+v", asked, back.Cases[0])
} }
+15
View File
@@ -150,6 +150,11 @@ const (
// WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder // WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder
// writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published. // writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published.
WaitsFor = "waits for: " WaitsFor = "waits for: "
// Retake: a failure that passes by itself (a name lookup that timed out, a refused connection): the builder takes
// the key again after a backoff, at most MaxRetakes times (issue 1748). A build module opens every such failure with it.
Retake = "retakeable: "
// MaxRetakes is how many times the builder takes a key again by itself for a retakeable failure.
MaxRetakes = 3
) )
// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint // FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
@@ -159,6 +164,12 @@ func FailedUntilChanged(err string) bool {
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
} }
// Retakeable says whether a failed key's error opens with Retake: the builder takes it again by itself after a backoff,
// until BuilderBuild.Retakes reaches MaxRetakes. An error that is FailedUntilChanged, or opens with no prefix, is not.
func Retakeable(err string) bool {
return strings.HasPrefix(err, Retake) && !FailedUntilChanged(err)
}
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
// the build module's recipe version. // the build module's recipe version.
type ToolchainUsed struct { type ToolchainUsed struct {
@@ -180,6 +191,10 @@ type BuilderBuild struct {
Artifacts []BuiltArtifact `json:"artifacts,omitempty"` Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
// Retakes is how many times the builder took this key again by itself after a retakeable failure (Retake); it is kept
// on the key, so a builder that starts again reads it and the count survives a restart (issue 1748). A reader that
// does not know the field reads 0.
Retakes int `json:"retakes,omitempty"`
// Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry // Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry
// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500 // does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
// clause 6). A reader that does not know the field reads the key as queued. // clause 6). A reader that does not know the field reads the key as queued.
+27
View File
@@ -86,11 +86,38 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) {
{VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false}, {VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false},
{RefusedKind + "a package in go", false}, {RefusedKind + "a package in go", false},
{"go test: 2 failed", true}, {"go test: 2 failed", true},
{Retake + "lookup git.novox.be: i/o timeout", true},
} { } {
if got := FailedUntilChanged(c.err); got == c.again { if got := FailedUntilChanged(c.err); got == c.again {
t.Errorf("FailedUntilChanged(%q) = %v", c.err, got) t.Errorf("FailedUntilChanged(%q) = %v", c.err, got)
} }
} }
// A retakeable error opens with Retake and is not one that waits for the fingerprint; the count is on the key's value (issue 1748).
for _, c := range []struct {
err string
want bool
}{
{Retake + "lookup git.novox.be: i/o timeout", true},
{NotPinned + "x", false},
{Retake[:4] + " not the prefix", false},
{"go test: 2 failed", false},
{"", false},
{VersionTaken + Retake, false},
} {
if got := Retakeable(c.err); got != c.want {
t.Errorf("Retakeable(%q) = %v", c.err, got)
}
}
data, err := EncodeBuilderBuild(BuilderBuild{State: BuildFailed, Error: Retake + "x", Retakes: 2})
if err != nil {
t.Fatal(err)
}
if back, err := DecodeBuilderBuild(data); err != nil || back.Retakes != 2 {
t.Fatalf("decoded %+v, %v", back, err)
}
if Retake != "retakeable: " || MaxRetakes != 3 {
t.Fatal("the retake prefix or its bound changed")
}
if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " { if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " {
t.Fatal("a prefix changed its words") t.Fatal("a prefix changed its words")
} }
-41
View File
@@ -34,10 +34,6 @@ const (
CheckerKeptHistory = 1 CheckerKeptHistory = 1
) )
// CheckerInputVerb is the verb of the seat mesh-delivery-checker that serves a checking program's input by its digest
// (RunInput.Verb; asked RunInputAsk, answering RunInputAnswer; issue 1921, milestone 47/B0).
const CheckerInputVerb = "input"
// The test levels (ADR 0445 clause 2) and the states of a verdict. // The test levels (ADR 0445 clause 2) and the states of a verdict.
const ( const (
LevelUnit = "unit" LevelUnit = "unit"
@@ -183,40 +179,6 @@ type Verdict struct {
// carried over, by repository (ADR 0548 §5): the verdict was rewritten for Trunk and Beside by running only the packages // carried over, by repository (ADR 0548 §5): the verdict was rewritten for Trunk and Beside by running only the packages
// whose digest the moves changed. Absent from a verdict judged whole, and from one still running. // whose digest the moves changed. Absent from a verdict judged whole, and from one still running.
CarriedFrom map[string]string `json:"carried-from,omitempty"` CarriedFrom map[string]string `json:"carried-from,omitempty"`
// JudgedBy names the checking program that judged the check: where it was built, the key its build is kept under and
// the heads it read (novox/hq ADR 0644, ADR 0656; design 47's bootstrap delivery, *Its tool*). Absent in a value
// written before it.
JudgedBy *JudgedBy `json:"judged-by,omitempty"`
}
// JudgedBy is the checking program's build that judged a check: built at Program, kept under Key (RunKey), Built when
// this check's run built it rather than taking the kept one, over Heads.
type JudgedBy struct {
Program RunProgram `json:"program"`
Key string `json:"key"`
Built bool `json:"built,omitempty"`
Heads []RunHead `json:"heads"`
}
func (j *JudgedBy) check() error {
if j == nil {
return nil
}
if err := checkRepository(j.Program.Repository); err != nil || !askCommitOK.MatchString(j.Program.Commit) || j.Program.Package == "" {
return fmt.Errorf("delivery: the verdict is judged by a checking program built at no repository, full commit and package (%+v)", j.Program)
}
if !strings.HasPrefix(j.Key, "program-") {
return fmt.Errorf("delivery: the verdict's checking program is kept under %q, which is no program key", j.Key)
}
if len(j.Heads) == 0 {
return fmt.Errorf("delivery: the verdict's checking program read no head")
}
for _, h := range j.Heads {
if err := checkRepository(h.Repository); err != nil || !askCommitOK.MatchString(h.Commit) {
return fmt.Errorf("delivery: the verdict's checking program read %s at %q, which is no full commit", h.Repository, h.Commit)
}
}
return nil
} }
func (c *Composition) check() error { func (c *Composition) check() error {
@@ -292,9 +254,6 @@ func (v Verdict) check() error {
if err := v.Composition.check(); err != nil { if err := v.Composition.check(); err != nil {
return err return err
} }
if err := v.JudgedBy.check(); err != nil {
return err
}
seen := map[string]bool{} seen := map[string]bool{}
for _, level := range v.OverBound { for _, level := range v.OverBound {
if err := checkLevel(level); err != nil { if err := checkLevel(level); err != nil {
+1 -18
View File
@@ -3,7 +3,6 @@ package delivery
import ( import (
"encoding/json" "encoding/json"
"errors" "errors"
"reflect"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
@@ -315,11 +314,6 @@ func TestVerdictNewFieldsChecked(t *testing.T) {
"carried from an empty commit": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": ""} }, []string{"carried-from", "mesh-sdk"}}, "carried from an empty commit": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": ""} }, []string{"carried-from", "mesh-sdk"}},
"carried over no move": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": "aaaa"} }, []string{"nothing moved"}}, "carried over no move": {func(v *Verdict) { v.CarriedFrom = map[string]string{"novox/mesh-sdk": "aaaa"} }, []string{"nothing moved"}},
"a running verdict carried": {func(v *Verdict) { v.State, v.CarriedFrom = VerdictRunning, map[string]string{"novox/mesh-sdk": "9999"} }, []string{"running", "carried"}}, "a running verdict carried": {func(v *Verdict) { v.State, v.CarriedFrom = VerdictRunning, map[string]string{"novox/mesh-sdk": "9999"} }, []string{"running", "carried"}},
// 1921 (B0): the checking program that judged it.
"judged by no program": {func(v *Verdict) { v.JudgedBy = &JudgedBy{Key: "program-0123456789abcdef", Heads: judgedBy().Heads} }, []string{"checking program", "full commit"}},
"judged under no key": {func(v *Verdict) { j := judgedBy(); j.Key = "composer-1"; v.JudgedBy = &j }, []string{"composer-1"}},
"judged over no head": {func(v *Verdict) { j := judgedBy(); j.Heads = nil; v.JudgedBy = &j }, []string{"no head"}},
"judged over a branch": {func(v *Verdict) { j := judgedBy(); j.Heads[0].Commit = "main"; v.JudgedBy = &j }, []string{"novox/mesh-delivery", "main"}},
} }
for name, c := range cases { for name, c := range cases {
v := passVerdict() v := passVerdict()
@@ -344,7 +338,7 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("the new reader of an old value: %v", err) t.Fatalf("the new reader of an old value: %v", err)
} }
if v.Composition != nil || v.OverBound != nil || v.Beside != nil || v.CarriedFrom != nil || v.JudgedBy != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 { if v.Composition != nil || v.OverBound != nil || v.Beside != nil || v.CarriedFrom != nil || v.Levels[LevelUnit].Skipped != 0 || v.Levels[LevelUnit].Run != 12 {
t.Errorf("an old value grew fields: %+v", v) t.Errorf("an old value grew fields: %+v", v)
} }
again, err := EncodeVerdict(v) again, err := EncodeVerdict(v)
@@ -358,16 +352,10 @@ func TestVerdictOldAndNewReadersAgree(t *testing.T) {
n.Beside = map[string]string{"novox/mesh-catalog": "cccc"} n.Beside = map[string]string{"novox/mesh-catalog": "cccc"}
n.CarriedFrom = map[string]string{"novox/mesh-sdk": "9999"} n.CarriedFrom = map[string]string{"novox/mesh-sdk": "9999"}
n.Composition.FPSet = "fps1:00" n.Composition.FPSet = "fps1:00"
j := judgedBy()
n.JudgedBy = &j
data, err := EncodeVerdict(n) data, err := EncodeVerdict(n)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// 1921 (B0): who judged it reads back whole.
if back, err := DecodeVerdict(data); err != nil || back.JudgedBy == nil || !reflect.DeepEqual(*back.JudgedBy, j) {
t.Errorf("judged by, read back: %+v %v", back.JudgedBy, err)
}
var ov oldVerdict var ov oldVerdict
if err := json.Unmarshal(data, &ov); err != nil || ov.Group != "feat/x" || ov.State != VerdictPass || ov.Levels[LevelUnit].Run != 12 || ov.SDK == "" { if err := json.Unmarshal(data, &ov); err != nil || ov.Group != "feat/x" || ov.State != VerdictPass || ov.Levels[LevelUnit].Run != 12 || ov.SDK == "" {
t.Errorf("the old reader of a new value: %+v, %v", ov, err) t.Errorf("the old reader of a new value: %+v, %v", ov, err)
@@ -670,8 +658,3 @@ func TestVerdictNamesEveryFailure(t *testing.T) {
t.Error("a verdict naming more than MaxFailures and the rest was written") t.Error("a verdict naming more than MaxFailures and the rest was written")
} }
} }
func judgedBy() JudgedBy {
return JudgedBy{Program: RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", Package: "p"},
Key: "program-0123456789abcdef", Built: true, Heads: []RunHead{{Repository: "novox/mesh-delivery", Commit: askCommit}}}
}
+3
View File
@@ -175,6 +175,9 @@ type BuildLogEntry struct {
// sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer // sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer
// keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's. // keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's.
Imports []string `json:"imports,omitempty"` Imports []string `json:"imports,omitempty"`
// Retakes is, at a build's end, how many times the builder took the key again by itself after a retakeable failure
// (BuilderBuild.Retakes; issue 1748): builder_builds_since reads it.
Retakes int `json:"retakes,omitempty"`
} }
// BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held // BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held
+7 -24
View File
@@ -237,9 +237,8 @@ func checkAskID(id string) error {
} }
// StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job. // StepAsk is the message on a build seat's stream: who asked what, of which kind and language, and the one whole job.
// Exactly one of Build, Check, Compose and Run is set: a delivery ask is a build; a check ask a build (a candidate's // Exactly one of Build, Check and Compose is set: a delivery ask is a build; a check ask a build (a candidate's
// fingerprint), a check or, on the Go seat, a compose or a run (RunJob); a recheck ask a check; an image ask is always a // fingerprint), a check or, on the Go seat, a compose; a recheck ask a check; an image ask is always a build.
// build.
type StepAsk struct { type StepAsk struct {
V int `json:"v"` V int `json:"v"`
ID string `json:"id"` ID string `json:"id"`
@@ -252,7 +251,6 @@ type StepAsk struct {
Build *BuildJob `json:"build,omitempty"` Build *BuildJob `json:"build,omitempty"`
Check *CheckJob `json:"check,omitempty"` Check *CheckJob `json:"check,omitempty"`
Compose *ComposeJob `json:"compose,omitempty"` Compose *ComposeJob `json:"compose,omitempty"`
Run *RunJob `json:"run,omitempty"`
// Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node. // Needs are what the node that runs the ask must have (AskNeeds); a check ask's alone. Empty: any node.
Needs []string `json:"needs,omitempty"` Needs []string `json:"needs,omitempty"`
// Packages are the version of each mesh package the build or the check builds against, by its module path (a Go // Packages are the version of each mesh package the build or the check builds against, by its module path (a Go
@@ -312,19 +310,9 @@ type ComposeAnswer struct {
// ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it // ComposerKey is the key a build module keeps a built composer under: the repository, commit, directory and package it
// was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once // was built from and the toolchain it was built with, hashed (`composer-` and 16 hex digits), so each head is built once
// on a node (ADR 0482 clause 4). The mesh package versions the ask names are an input of the build, so they are in the // on a node (ADR 0482 clause 4).
// key, sorted, when there are any; an ask that names none gets the key it always had (issue 1921). func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed) string {
func ComposerKey(repository, commit, dir, pkg string, toolchain ToolchainUsed, versions PackageVersions) string { sum := sha256.Sum256([]byte(strings.Join([]string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}, "\n")))
parts := []string{repository, commit, dir, pkg, toolchain.Version, toolchain.Recipe}
pkgs := make([]string, 0, len(versions))
for p := range versions {
pkgs = append(pkgs, p)
}
sort.Strings(pkgs)
for _, p := range pkgs {
parts = append(parts, p+"="+versions[p])
}
sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
return "composer-" + hex.EncodeToString(sum[:8]) return "composer-" + hex.EncodeToString(sum[:8])
} }
@@ -411,18 +399,16 @@ func (a StepAsk) check() error {
return fmt.Errorf("delivery: the ask %s names no time", a.ID) return fmt.Errorf("delivery: the ask %s names no time", a.ID)
} }
jobs := 0 jobs := 0
for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil, a.Run != nil} { for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} {
if set { if set {
jobs++ jobs++
} }
} }
switch { switch {
case jobs != 1: case jobs != 1:
return fmt.Errorf("delivery: the ask %s holds a build, a check, a compose or a run, exactly one", a.ID) return fmt.Errorf("delivery: the ask %s holds a build, a check or a compose, exactly one", a.ID)
case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo): case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo):
return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID) return fmt.Errorf("delivery: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID)
case a.Run != nil && (a.Kind != AskCheck || a.Language != AskGo):
return fmt.Errorf("delivery: the ask %s is a run, which the Go seat's check stream alone carries", a.ID)
case a.Build != nil && a.Kind == AskRecheck: case a.Build != nil && a.Kind == AskRecheck:
return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) return fmt.Errorf("delivery: the ask %s is a build on the recheck stream, which carries checks alone", a.ID)
case a.Check != nil && a.Kind == AskDelivery: case a.Check != nil && a.Kind == AskDelivery:
@@ -448,9 +434,6 @@ func (a StepAsk) check() error {
if a.Compose != nil { if a.Compose != nil {
return a.Compose.check(a.ID) return a.Compose.check(a.ID)
} }
if a.Run != nil {
return a.Run.check(a.ID)
}
return a.Check.check(a.ID) return a.Check.check(a.ID)
} }
+19 -125
View File
@@ -2,7 +2,6 @@ package delivery
import ( import (
"encoding/json" "encoding/json"
"errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@@ -17,36 +16,20 @@ func composeAsk() StepAsk {
Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}} Request: json.RawMessage(`{"v":1,"cases":[{"name":"change","manifests":{"demo":{"digest":"d1"}}}]}`), Reader: "mesh-sdk head"}}
} }
// runAsk is the checker's run of a checking program built at a mesh-delivery head, over two heads (issue 1921, B0). // 1622: a compose ask encodes and decodes with every field.
func runAsk() StepAsk {
in, err := RunInputOf("seat:mesh-delivery-checker."+CheckerInputVerb, []byte(`{"v":1}`))
if err != nil {
panic(err)
}
return StepAsk{ID: "mh3k2x9qa1b2c6", Kind: AskCheck, Language: AskGo, Asker: "mesh-delivery-checker",
At: time.Date(2026, 11, 6, 22, 0, 0, 0, time.UTC),
Run: &RunJob{Program: RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker",
Package: "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/cmd/mesh-checking-program"},
Heads: []RunHead{{Repository: "novox/mesh-delivery", Commit: askCommit}, {Repository: "novox/mesh-sdk", Commit: askCommit}},
Input: in, Args: []string{"-verdict"}, Reader: "checking program of mesh-delivery head"},
Packages: PackageVersions{"git.novox.be/novox/mesh-sdk/go": askCommit}}
}
// 1622: a compose ask encodes and decodes with every field; 1921 (B0): a run ask too.
func TestAComposeAskRoundTrips(t *testing.T) { func TestAComposeAskRoundTrips(t *testing.T) {
for _, a := range []StepAsk{composeAsk(), runAsk()} { a := composeAsk()
raw, err := EncodeStepAsk(a) raw, err := EncodeStepAsk(a)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
got, err := DecodeStepAsk(raw) got, err := DecodeStepAsk(raw)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
a.V = StepAskVersion a.V = StepAskVersion
if !reflect.DeepEqual(got, a) { if !reflect.DeepEqual(got, a) {
t.Fatalf("round trip:\n got %+v\nwant %+v", got, a) t.Fatalf("round trip:\n got %+v\nwant %+v", got, a)
}
} }
} }
@@ -71,112 +54,23 @@ func TestAComposeAskIsRefusedWhereItDoesNotBelong(t *testing.T) {
t.Errorf("a compose ask %s is encoded", name) t.Errorf("a compose ask %s is encoded", name)
} }
} }
// 1921 (B0): a run ask likewise goes only on the Go seat's check stream, alone, builds a clean package at a full
// commit, reads each named head once at a full commit, and names an input kept by digest within its bound.
for name, change := range map[string]func(*StepAsk){
"on the delivery stream": func(a *StepAsk) { a.Kind = AskDelivery },
"on the Node seat": func(a *StepAsk) { a.Language = AskNode },
"beside a compose": func(a *StepAsk) { a.Compose = composeAsk().Compose },
"with a short commit": func(a *StepAsk) { a.Run.Program.Commit = "0123456" },
"outside the repository": func(a *StepAsk) { a.Run.Program.Dir = "../x" },
"with no package": func(a *StepAsk) { a.Run.Program.Package = "" },
"with no head": func(a *StepAsk) { a.Run.Heads = nil },
"with a head twice": func(a *StepAsk) { a.Run.Heads = append(a.Run.Heads, a.Run.Heads[0]) },
"with a head at a branch": func(a *StepAsk) { a.Run.Heads[1].Commit = "main" },
"with an input on a state": func(a *StepAsk) { a.Run.Input.Verb = "mesh-delivery-checker.inputs" },
"with an input on one machine": func(a *StepAsk) { a.Run.Input.Verb = "seat:mesh-delivery-checker.input@ace" },
"with an input by no digest": func(a *StepAsk) { a.Run.Input.Digest = "d1" },
"with an input over its bound": func(a *StepAsk) { a.Run.Input.Size = RunInputBound + 1 },
"with an argument of the shell": func(a *StepAsk) { a.Run.Args = []string{"-x; rm"} },
} {
a := runAsk()
change(&a)
if _, err := EncodeStepAsk(a); err == nil {
t.Errorf("a run ask %s is encoded", name)
}
}
// The asker refuses an input over its bound before it asks, naming its size; the build module refuses bytes that
// are not the input named.
if _, err := RunInputOf("seat:mesh-delivery-checker.input", make([]byte, RunInputBound+1)); err == nil || !strings.Contains(err.Error(), "262145 bytes") {
t.Errorf("an input over its bound: %v", err)
}
in := runAsk().Run.Input
if err := in.Verify([]byte(`{"v":1}`)); err != nil {
t.Errorf("the input named: %v", err)
}
if err := in.Verify([]byte(`{"v":2}`)); err == nil {
t.Error("another input read as the one named")
}
// The build module asks the verb the input names for its digest and takes only the bytes named (the operator's answer
// on B0: the input travels by a verb of the asker's seat, never a read of its state).
for name, c := range map[string]struct {
answer string
err error
want string
}{
"the bytes named": {answer: `{"digest":"` + in.Digest + `","input":"eyJ2IjoxfQ=="}`},
"other bytes": {answer: `{"digest":"` + in.Digest + `","input":"eyJ2IjoyfQ=="}`, want: "is not the one the ask names"},
"another digest": {answer: `{"digest":"` + strings.Repeat("0", 64) + `","input":"eyJ2IjoxfQ=="}`, want: "answered the input"},
"no input": {answer: `[1]`, want: "answered no input"},
"an error of the callee": {err: errors.New("keeps no input of that digest"), want: "keeps no input of that digest"},
} {
var asked string
var body any
got, err := in.Read(func(key string, b any) (json.RawMessage, error) {
asked, body = key, b
return json.RawMessage(c.answer), c.err
})
if asked != "seat:mesh-delivery-checker.input" || body != (RunInputAsk{Digest: in.Digest}) {
t.Errorf("%s: asked %s with %+v", name, asked, body)
}
switch {
case c.want == "" && (err != nil || string(got) != `{"v":1}`):
t.Errorf("%s: %q, %v", name, got, err)
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want)):
t.Errorf("%s: %v, want %q", name, err, c.want)
}
}
} }
// 1622/AC3, 1921: a composer is kept under one key per repository, commit, directory, package, toolchain and asked versions, so two asks of // 1622/AC3: a composer is kept under one key per repository, commit, directory, package and toolchain, so two asks of
// one head build it once on a node, and another head or toolchain builds its own. // one head build it once on a node, and another head or toolchain builds its own.
func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) { func TestAComposerIsKeptByWhatItWasBuiltFrom(t *testing.T) {
tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"} tc := ToolchainUsed{Version: "go1.26.0", Recipe: "3"}
k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, nil) k := ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc)
if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, nil) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 { if k != ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc) || !strings.HasPrefix(k, "composer-") || len(k) != len("composer-")+16 {
t.Fatalf("key = %q", k) t.Fatalf("key = %q", k)
} }
for _, other := range []string{ for _, other := range []string{
ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc, nil), ComposerKey("novox/mesh-sdk", strings.Repeat("1", 40), "go", "p", tc),
ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc, nil), ComposerKey("novox/mesh-controller", askCommit, ".", "p", tc),
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}, nil), ComposerKey("novox/mesh-sdk", askCommit, "go", "p", ToolchainUsed{Version: "go1.26.1", Recipe: "3"}),
ComposerKey("novox/mesh-sdk", askCommit, "go", "p", tc, PackageVersions{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("2", 40)}),
} { } {
if other == k { if other == k {
t.Errorf("two composers built from different inputs share the key %s", k) t.Errorf("two composers built from different inputs share the key %s", k)
} }
} }
// 1921: the asked versions are in the key, in any order of naming; none, or an empty ask, gives the key without them.
a, b := strings.Repeat("a", 40), strings.Repeat("b", 40)
k1 := ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/x": a, "m/y": b})
if k1 != ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/y": b, "m/x": a}) {
t.Error("the key depends on the order of the versions")
}
if k1 == ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{"m/x": b, "m/y": a}) || k1 == ComposerKey("r", askCommit, "go", "p", tc, nil) {
t.Error("other versions, or none, share the key")
}
if ComposerKey("r", askCommit, "go", "p", tc, nil) != ComposerKey("r", askCommit, "go", "p", tc, PackageVersions{}) {
t.Error("an empty ask changed the key")
}
// 1921 (B0): a run's program is kept the same way, under a key of its own kind.
p := RunProgram{Repository: "novox/mesh-delivery", Commit: askCommit, Dir: "modules/mesh-delivery-checker", Package: "p"}
rk := RunKey(p, tc, nil)
if !strings.HasPrefix(rk, "program-") || len(rk) != len("program-")+16 || rk != RunKey(p, tc, nil) {
t.Errorf("run key = %q", rk)
}
q := p
q.Commit = strings.Repeat("1", 40)
if RunKey(q, tc, nil) == rk || RunKey(p, tc, PackageVersions{"m/x": a}) == rk {
t.Error("two programs built from different inputs share the key")
}
} }
-201
View File
@@ -1,201 +0,0 @@
package delivery
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path"
"regexp"
"strings"
"git.novox.be/novox/mesh-sdk/go/calladdress"
)
// **A run ask has a program built from a change and run in the build seat's sandbox** (novox/hq ADR 0644, ADR 0656;
// design 47, *What ADR 0656 leaves to this design*: the build seat's ask `run`; issue 1921). It goes on the Go seat's
// check stream, a whole job pulled where there is capacity, like a compose: the build module fetches Program at its
// commit and builds it once per RunKey, in a step that runs nothing but the build; fetches every head of Heads and lays
// each read-only under the directory RunHeadsEnv names, at its repository's path (`<owner>/<name>`); reads the input
// Input names, checks its digest and writes it to the file RunInputEnv names; then runs the program with Args, no
// network and no bus account (ADR 0448 §2, §4), and answers RunAnswer on its step key.
//
// The input is carried by reference, as a compose carries its manifests: the asker keeps it by its digest, measured
// against RunInputBound before it asks, and serves it on a verb of its seat (RunInput.Verb, answered RunInputAnswer); the
// build module asks that verb as its manifest's `invokes` grants it, never reading the asker's state (ADR 0596 §2; the
// operator's answer on milestone 47/B0), so the ask on the stream never carries it (issue 1930).
// RunJob is a run ask's job.
type RunJob struct {
// Program is where the program is built: the repository and full commit fetched, the Go module's directory in it and
// the program's import path. The ask's Packages are the mesh package versions it builds against (ADR 0573 §2, §4).
Program RunProgram `json:"program"`
// Heads are the commits the program reads, each laid read-only at its repository's path; a repository at most once.
Heads []RunHead `json:"heads"`
// Input names the program's input, kept by the asker by its digest and served on a verb of its seat.
Input RunInput `json:"input"`
// Args are the program's arguments (a checking program's `-seats` or `-verdict`), each a word or a flag.
Args []string `json:"args,omitempty"`
// Reader names, for a reader of the ask, what the run is for ("checking program of mesh-delivery head").
Reader string `json:"reader,omitempty"`
}
// RunProgram is where a run's program is built from.
type RunProgram struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Dir string `json:"dir"`
Package string `json:"package"`
}
// RunHead is one commit a run's program reads.
type RunHead struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
}
// RunInput is a run's input by reference: the seat verb the asker serves it on, named as a manifest's `invokes` names it
// (`seat:<seat>.<verb>`, asked of no one machine), its digest (the sha256 of its bytes, 64 lowercase hex digits) and its
// size in bytes. The verb is asked RunInputAsk and answers RunInputAnswer.
type RunInput struct {
Verb string `json:"verb"`
Digest string `json:"digest"`
Size int `json:"size"`
}
// RunInputAsk is what a verb a RunInput names is asked: the input's digest.
type RunInputAsk struct {
Digest string `json:"digest"`
}
// RunInputAnswer is that verb's answer: the digest asked and the input's exact bytes (base64 in JSON, so no encoding on
// the way changes a byte the digest covers).
type RunInputAnswer struct {
Digest string `json:"digest"`
Input []byte `json:"input"`
}
// RunInputBound is the most bytes a run's input may hold: the bus's value limit the design already uses (256 KiB), so
// the asker keeps it as one value (the lead's ruling on milestone 47/B0, 2026-11-06).
const RunInputBound = 256 << 10
// RunHeadsEnv names, for the program, the directory its heads are laid in; RunInputEnv the file its input is written to.
const (
RunHeadsEnv = "MESH_RUN_HEADS"
RunInputEnv = "MESH_RUN_INPUT"
)
// RunInputOf is the reference to body served on verb (`seat:<seat>.<verb>`), refusing a body over RunInputBound, naming
// its size, so an asker refuses it before asking.
func RunInputOf(verb string, body []byte) (RunInput, error) {
if len(body) > RunInputBound {
return RunInput{}, fmt.Errorf("delivery: a run's input of %d bytes is over its bound of %d bytes", len(body), RunInputBound)
}
sum := sha256.Sum256(body)
in := RunInput{Verb: verb, Digest: hex.EncodeToString(sum[:]), Size: len(body)}
return in, in.check("")
}
// Verify refuses body when it is not the input i names: another size or another digest.
func (i RunInput) Verify(body []byte) error {
sum := sha256.Sum256(body)
if len(body) != i.Size || hex.EncodeToString(sum[:]) != i.Digest {
return fmt.Errorf("delivery: the input %s answered for %s is not the one the ask names (%d bytes, %d named)", i.Verb, i.Digest, len(body), i.Size)
}
return nil
}
// Read asks the verb i names for its input with ask (a tools bundle's stdio.Ask, a process's Conn.Call) and answers its
// bytes, refusing an answer for another digest or bytes that are not the input named.
func (i RunInput) Read(ask func(key string, body any) (json.RawMessage, error)) ([]byte, error) {
raw, err := ask(i.Verb, RunInputAsk{Digest: i.Digest})
if err != nil {
return nil, fmt.Errorf("delivery: asking %s for the input %s: %w", i.Verb, i.Digest, err)
}
var a RunInputAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("delivery: %s answered no input for %s: %w", i.Verb, i.Digest, err)
}
if a.Digest != i.Digest {
return nil, fmt.Errorf("delivery: %s answered the input %q, asked %s", i.Verb, a.Digest, i.Digest)
}
return a.Input, i.Verify(a.Input)
}
var (
runDigestOK = regexp.MustCompile(`^[0-9a-f]{64}$`)
runArgOK = regexp.MustCompile(`^[-=_./a-zA-Z0-9]+$`)
)
func (i RunInput) check(id string) error {
switch {
case !runVerbOK(i.Verb):
return fmt.Errorf("delivery: the run ask %s serves its input on %q, which is no seat:<seat>.<verb> of no one machine", id, i.Verb)
case !runDigestOK.MatchString(i.Digest):
return fmt.Errorf("delivery: the run ask %s names its input by %q, which is no sha256 digest", id, i.Digest)
case i.Size <= 0 || i.Size > RunInputBound:
return fmt.Errorf("delivery: the run ask %s names an input of %d bytes, outside 1 to %d", id, i.Size, RunInputBound)
}
return nil
}
// runVerbOK says key names a seat's verb, asked of no one machine.
func runVerbOK(key string) bool {
a, err := calladdress.Parse(key)
return err == nil && a.Seat != "" && a.Node == ""
}
func (r RunJob) check(id string) error {
p := r.Program
if err := checkRepository(p.Repository); err != nil {
return fmt.Errorf("the run ask %s's program: %w", id, err)
}
if !askCommitOK.MatchString(p.Commit) {
return fmt.Errorf("delivery: the run ask %s builds its program at no full commit", id)
}
if p.Dir == "" || p.Dir != path.Clean(p.Dir) || strings.HasPrefix(p.Dir, "/") || p.Dir == ".." || strings.HasPrefix(p.Dir, "../") {
return fmt.Errorf("delivery: the run ask %s names no clean directory inside the repository (%q)", id, p.Dir)
}
if p.Package == "" || strings.ContainsAny(p.Package, " \t\n") {
return fmt.Errorf("delivery: the run ask %s names no package to build (%q)", id, p.Package)
}
if len(r.Heads) == 0 {
return fmt.Errorf("delivery: the run ask %s names no head", id)
}
seen := map[string]bool{}
for _, h := range r.Heads {
if err := checkRepository(h.Repository); err != nil {
return fmt.Errorf("the run ask %s's head: %w", id, err)
}
if !askCommitOK.MatchString(h.Commit) {
return fmt.Errorf("delivery: the run ask %s names the head of %s at no full commit", id, h.Repository)
}
if seen[h.Repository] {
return fmt.Errorf("delivery: the run ask %s names %s twice", id, h.Repository)
}
seen[h.Repository] = true
}
for _, a := range r.Args {
if !runArgOK.MatchString(a) {
return fmt.Errorf("delivery: the run ask %s names the argument %q, which is no word or flag", id, a)
}
}
return r.Input.check(id)
}
// RunAnswer is a run ask's step result: whether this run built the program or took the one kept under its key, the
// key, the program's exit, the last line of its standard output (Output, raw when it is JSON, else as a JSON string),
// and Said: the first line of its standard error, or why the step could not run it.
type RunAnswer struct {
Built bool `json:"built"`
Key string `json:"key"`
Exit int `json:"exit"`
Output json.RawMessage `json:"output,omitempty"`
Said string `json:"said,omitempty"`
}
// RunKey is the key a build module keeps a built program under, as ComposerKey keeps a composer: what it was built
// from, the toolchain and the asked package versions, hashed (`program-` and 16 hex digits).
func RunKey(p RunProgram, toolchain ToolchainUsed, versions PackageVersions) string {
return "program-" + strings.TrimPrefix(ComposerKey(p.Repository, p.Commit, p.Dir, p.Package, toolchain, versions), "composer-")
}