BM1: the build modules give a step met away again by again-delays

Each build module declares again-delays (15s,60s) and gives its puller the
delays steprun.AgainDelays reads from it; a publish's no responders is a
transport fault by steprun.PublishFault with no seat call to learn why, so
the modules no longer invoke the assigner's assignments.

Builds: design 47 H2 §799; H-A-350 DMC-7, H-A-390
This commit is contained in:
2026-11-22 00:27:04 +01:00
parent eac56f9990
commit 41610e903f
13 changed files with 136 additions and 38 deletions
@@ -5,6 +5,8 @@ import (
"encoding/json"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -40,6 +42,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or download to its step's error say a network that did
// not answer as a transport fault, and a failure of the source (an unknown revision, a checksum) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
// A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its
// first run and 60 s after its second by default (design 47 H2 §799; H-A-390).
t.Run("again-delays is declared and read", func(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]json.RawMessage `json:"config"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
var c struct {
Default string `json:"default"`
Runtime bool `json:"runtime"`
}
decl, ok := m.Config[steprun.AgainDelaysSetting]
_ = json.Unmarshal(decl, &c)
delay, err := steprun.AgainDelays(c.Default)
if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second {
t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err)
}
})
conformance.Passing(t, []string{conformance.GitFetch, conformance.GoDownload}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
@@ -102,6 +102,14 @@ func main() {
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
// A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390).
againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting)
againDelay, againErr := steprun.AgainDelays(againSetting)
if againErr != nil {
logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr)
os.Exit(1)
}
puller.Delay = againDelay
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890): one that names another owner, or none and
@@ -17,7 +17,6 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
registry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
)
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
@@ -147,26 +146,13 @@ func TestARefusedPushFailsTheStepNotStored(t *testing.T) {
!strings.Contains(st.Reason, "oci-artifact-registry") {
t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason)
}
// ADR 0664 §1 (issue 1748): "no responders" is the same words for a seat nothing holds and for a holder whose tool runner is
// being replaced; the assigner says which. A named holder makes it a transport fault (the bus gives the ask again), none
// fails at once naming it.
for _, c := range []struct {
name, holders string
passing bool
words string
}{
{"a holder named", `{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, true, "the holder is silent: seat oci-artifact-registry, held by mesh-registry on novox"},
{"nothing holds it", `{"seats":[{"seat":"go-package-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, false, "nothing holds oci-artifact-registry"},
} {
m.Call = func(key string, _ any) (json.RawMessage, error) {
if key == steprun.AssignerHolders {
return json.RawMessage(c.holders), nil
}
return nil, errors.New("nats: no responders available for request")
}
if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || st.Passing != c.passing || !strings.Contains(st.Reason, c.words) {
t.Errorf("%s: the step ended %s: %q (passing %v)", c.name, st.State, st.Reason, st.Passing)
}
// A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1):
// the run ends again, its why opening "publish: ".
m.Call = func(key string, _ any) (json.RawMessage, error) {
return nil, errors.New("nats: no responders available for request")
}
if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !st.Passing || !strings.Contains(st.Reason, "publish: ") {
t.Errorf("the step ended %s: %q (passing %v)", st.State, st.Reason, st.Passing)
}
}
+7 -1
View File
@@ -45,7 +45,6 @@
"seat:go-package-registry.publish-part",
"seat:mesh-catalog.input",
"seat:mesh-wants.input",
"seat:mesh-module-assigner.assignments",
"seat:mesh-catalog.module",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-catalog.entry"
@@ -68,6 +67,13 @@
"description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 6
},
"again-delays": {
"type": "string",
"description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.",
"trusted": false,
"runtime": true,
"default": "15s,60s"
}
},
"tools": [
@@ -24,7 +24,7 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) {
t.Fatal(err)
}
if want := []string{"seat:mesh-hand-acts.hand-act", "seat:container-image-registry.publish", "seat:container-image-registry.publish-part",
"seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) {
"seat:container-image-registry.mirror", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// The seat's outcomes a copy waits on; the version comes in the ask, so version.numbered is not heard (design 47 G §413).
@@ -112,8 +112,8 @@ func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) {
raw, err := m.call()("seat:"+steprun.ImageSeat+"."+reg.VerbMirror, reg.MirrorInput{Name: module + "/" + name, Source: from, Digest: digest})
if err != nil {
said.Refused(err)
// A seat verb's "no responders" with a holder named is a transport fault (ADR 0664 §1): the seat's tool runner is replaced.
err = steprun.HolderSilentOr(m.call(), steprun.ImageSeat, err)
// A seat verb's "no responders" is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1).
err = steprun.PublishFault(err)
return nil, fmt.Errorf("%s%s: %s's mirror: %w", builds.NotStored, name, steprun.ImageSeat, err)
}
var answer reg.MirrorAnswer
@@ -7,6 +7,8 @@ import (
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -27,6 +29,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or image build to its step's error say a network that
// did not answer as a transport fault, and a failure of the source (a RUN line that fails) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
// A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its
// first run and 60 s after its second by default (design 47 H2 §799; H-A-390).
t.Run("again-delays is declared and read", func(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]json.RawMessage `json:"config"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
var c struct {
Default string `json:"default"`
Runtime bool `json:"runtime"`
}
decl, ok := m.Config[steprun.AgainDelaysSetting]
_ = json.Unmarshal(decl, &c)
delay, err := steprun.AgainDelays(c.Default)
if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second {
t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err)
}
})
conformance.Passing(t, []string{conformance.GitFetch, conformance.BuildahBud}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
@@ -71,6 +71,14 @@ func main() {
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
// A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390).
againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting)
againDelay, againErr := steprun.AgainDelays(againSetting)
if againErr != nil {
logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr)
os.Exit(1)
}
puller.Delay = againDelay
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890): one that names another owner, or none and
+7 -1
View File
@@ -42,6 +42,13 @@
"description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 2
},
"again-delays": {
"type": "string",
"description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.",
"trusted": false,
"runtime": true,
"default": "15s,60s"
}
},
"tools": [
@@ -67,7 +74,6 @@
"seat:container-image-registry.publish",
"seat:container-image-registry.publish-part",
"seat:container-image-registry.mirror",
"seat:mesh-module-assigner.assignments",
"seat:mesh-catalog.module",
"seat:mesh-catalog.seat-definitions"
],
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -37,6 +39,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
// ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or install to its step's error say a network that did
// not answer as a transport fault, and a failure of the source (a package the registry lacks) as none.
func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) {
// A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its
// first run and 60 s after its second by default (design 47 H2 §799; H-A-390).
t.Run("again-delays is declared and read", func(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]json.RawMessage `json:"config"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
var c struct {
Default string `json:"default"`
Runtime bool `json:"runtime"`
}
decl, ok := m.Config[steprun.AgainDelaysSetting]
_ = json.Unmarshal(decl, &c)
delay, err := steprun.AgainDelays(c.Default)
if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second {
t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err)
}
})
conformance.Passing(t, []string{conformance.GitFetch, conformance.NpmCI}, func(runner, words string) error {
switch runner {
case conformance.GitFetch:
@@ -83,6 +83,14 @@ func main() {
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
// A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390).
againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting)
againDelay, againErr := steprun.AgainDelays(againSetting)
if againErr != nil {
logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr)
os.Exit(1)
}
puller.Delay = againDelay
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890): one that names another owner, or none and
@@ -137,17 +137,9 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) {
if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), builds.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) {
t.Errorf("a refused publish answered %v", err)
}
if c.noSeat {
// ADR 0664 §1: with a holder named by the assigner the same words are a transport fault, the bus gives the ask again.
m.Call = func(key string, _ any) (json.RawMessage, error) {
if key == steprun.AssignerHolders {
return json.RawMessage(`{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`), nil
}
return nil, errors.New("nats: no responders available for request")
}
if _, err := m.build(context.Background(), buildArgs()); err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "the holder is silent") {
t.Errorf("no responders with a holder named answered %v, want a transport fault", err)
}
if c.noSeat && (err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "publish: ")) {
// A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799).
t.Errorf("no responders answered %v, want a transport fault opening publish:", err)
}
var archives []string
filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error {
+7 -1
View File
@@ -46,6 +46,13 @@
"description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.",
"minimum": 0.1,
"default": 2
},
"again-delays": {
"type": "string",
"description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.",
"trusted": false,
"runtime": true,
"default": "15s,60s"
}
},
"tools": [
@@ -79,7 +86,6 @@
"seat:oci-artifact-registry.publish-part",
"seat:npm-package-registry.publish",
"seat:npm-package-registry.publish-part",
"seat:mesh-module-assigner.assignments",
"seat:mesh-catalog.module",
"seat:mesh-catalog.seat-definitions"
],