From 41610e903f4ed4ae3217967e8da51a06d70e6fc4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:27:04 +0100 Subject: [PATCH] BM1: the build modules give a step met away again by again-delays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each build module declares again-delays (15s,60s) and gives its puller the delays steprun.AgainDelays reads from it; a publish's no responders is a transport fault by steprun.PublishFault with no seat call to learn why, so the modules no longer invoke the assigner's assignments. Builds: design 47 H2 §799; H-A-350 DMC-7, H-A-390 --- .../cmd/mesh-build-go/kit_test.go | 26 +++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/main.go | 8 ++++++ .../cmd/mesh-build-go/publish_test.go | 28 +++++-------------- modules/mesh-build-go/module.json | 8 +++++- .../cmd/mesh-build-image/compose_test.go | 2 +- .../cmd/mesh-build-image/copy.go | 4 +-- .../cmd/mesh-build-image/kit_test.go | 26 +++++++++++++++++ .../cmd/mesh-build-image/main.go | 8 ++++++ modules/mesh-build-image/module.json | 8 +++++- .../cmd/mesh-build-node/kit_test.go | 26 +++++++++++++++++ .../cmd/mesh-build-node/main.go | 8 ++++++ .../cmd/mesh-build-node/push_test.go | 14 ++-------- modules/mesh-build-node/module.json | 8 +++++- 13 files changed, 136 insertions(+), 38 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go index 4cbf1de79..3f1962c7d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/kit_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/forgekey" + "os" + "path/filepath" "strings" "testing" "time" @@ -40,6 +42,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or download to its step's error say a network that did // not answer as a transport fault, and a failure of the source (an unknown revision, a checksum) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.GoDownload}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 7886aaca6..79a538789 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -102,6 +102,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go index 92b0cc160..06ac02449 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -17,7 +17,6 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" registry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" ) // pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. @@ -147,26 +146,13 @@ func TestARefusedPushFailsTheStepNotStored(t *testing.T) { !strings.Contains(st.Reason, "oci-artifact-registry") { t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason) } - // ADR 0664 §1 (issue 1748): "no responders" is the same words for a seat nothing holds and for a holder whose tool runner is - // being replaced; the assigner says which. A named holder makes it a transport fault (the bus gives the ask again), none - // fails at once naming it. - for _, c := range []struct { - name, holders string - passing bool - words string - }{ - {"a holder named", `{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, true, "the holder is silent: seat oci-artifact-registry, held by mesh-registry on novox"}, - {"nothing holds it", `{"seats":[{"seat":"go-package-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`, false, "nothing holds oci-artifact-registry"}, - } { - m.Call = func(key string, _ any) (json.RawMessage, error) { - if key == steprun.AssignerHolders { - return json.RawMessage(c.holders), nil - } - return nil, errors.New("nats: no responders available for request") - } - if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || st.Passing != c.passing || !strings.Contains(st.Reason, c.words) { - t.Errorf("%s: the step ended %s: %q (passing %v)", c.name, st.State, st.Reason, st.Passing) - } + // A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1): + // the run ends again, its why opening "publish: ". + m.Call = func(key string, _ any) (json.RawMessage, error) { + return nil, errors.New("nats: no responders available for request") + } + if _, st := runBuild(t, m, states, "tools"); st.State != sdkstep.StepFailed || !st.Passing || !strings.Contains(st.Reason, "publish: ") { + t.Errorf("the step ended %s: %q (passing %v)", st.State, st.Reason, st.Passing) } } diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 48acd5e4f..3198c8f59 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -45,7 +45,6 @@ "seat:go-package-registry.publish-part", "seat:mesh-catalog.input", "seat:mesh-wants.input", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.entry" @@ -68,6 +67,13 @@ "description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 6 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index e06d84c2d..5dd0fc2f3 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -24,7 +24,7 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { t.Fatal(err) } if want := []string{"seat:mesh-hand-acts.hand-act", "seat:container-image-registry.publish", "seat:container-image-registry.publish-part", - "seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) { + "seat:container-image-registry.mirror", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions"}; !slices.Equal(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The seat's outcomes a copy waits on; the version comes in the ask, so version.numbered is not heard (design 47 G §413). diff --git a/modules/mesh-build-image/cmd/mesh-build-image/copy.go b/modules/mesh-build-image/cmd/mesh-build-image/copy.go index 9a09ca637..48a4da86f 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/copy.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/copy.go @@ -112,8 +112,8 @@ func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) { raw, err := m.call()("seat:"+steprun.ImageSeat+"."+reg.VerbMirror, reg.MirrorInput{Name: module + "/" + name, Source: from, Digest: digest}) if err != nil { said.Refused(err) - // A seat verb's "no responders" with a holder named is a transport fault (ADR 0664 §1): the seat's tool runner is replaced. - err = steprun.HolderSilentOr(m.call(), steprun.ImageSeat, err) + // A seat verb's "no responders" is a transport fault with no seat call to learn why (design 47 H2 §799, ADR 0664 §1). + err = steprun.PublishFault(err) return nil, fmt.Errorf("%s%s: %s's mirror: %w", builds.NotStored, name, steprun.ImageSeat, err) } var answer reg.MirrorAnswer diff --git a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go index 1dea213b0..d151dcf28 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/kit_test.go @@ -7,6 +7,8 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/forgekey" + "os" + "path/filepath" "strings" "testing" "time" @@ -27,6 +29,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or image build to its step's error say a network that // did not answer as a transport fault, and a failure of the source (a RUN line that fails) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.BuildahBud}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 47a0bfd23..61b05a9c8 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -71,6 +71,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 9493c8927..6f99328dc 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -42,6 +42,13 @@ "description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 2 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ @@ -67,7 +74,6 @@ "seat:container-image-registry.publish", "seat:container-image-registry.publish-part", "seat:container-image-registry.mirror", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions" ], diff --git a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go index a9dc94efb..68d5e34a3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/kit_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "os" + "path/filepath" "strings" "testing" "time" @@ -37,6 +39,30 @@ func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) { // ADR 0664 §1 (issue 1748): the module's own paths from a failed fetch or install to its step's error say a network that did // not answer as a transport fault, and a failure of the source (a package the registry lacks) as none. func TestTheStepsSayAFailedNetworkAsATransportFault(t *testing.T) { + // A step whose fetch or publish met its holder away is given again after the module's again-delays, 15 s after its + // first run and 60 s after its second by default (design 47 H2 §799; H-A-390). + t.Run("again-delays is declared and read", func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var c struct { + Default string `json:"default"` + Runtime bool `json:"runtime"` + } + decl, ok := m.Config[steprun.AgainDelaysSetting] + _ = json.Unmarshal(decl, &c) + delay, err := steprun.AgainDelays(c.Default) + if !ok || !c.Runtime || err != nil || delay(1) != 15*time.Second || delay(2) != 60*time.Second { + t.Fatalf("again-delays declared %v as %+v: %v", ok, c, err) + } + }) conformance.Passing(t, []string{conformance.GitFetch, conformance.NpmCI}, func(runner, words string) error { switch runner { case conformance.GitFetch: diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 18752e4d7..92a9ca911 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -83,6 +83,14 @@ func main() { steps := m.StepTools() booted := time.Now() puller := m.Puller(steps, nil) + // A step whose fetch or publish met its holder away is given again after again-delays (design 47 H2 §799; H-A-390). + againSetting, _ := stdio.Setting(steprun.AgainDelaysSetting) + againDelay, againErr := steprun.AgainDelays(againSetting) + if againErr != nil { + logf("the setting %s: %v", steprun.AgainDelaysSetting, againErr) + os.Exit(1) + } + puller.Delay = againDelay m.Held = puller if m.Node != "" { // A step a previous process left running is ended (issue 1890): one that names another owner, or none and diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index 5e21c8905..fa5b44c01 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -137,17 +137,9 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), builds.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) { t.Errorf("a refused publish answered %v", err) } - if c.noSeat { - // ADR 0664 §1: with a holder named by the assigner the same words are a transport fault, the bus gives the ask again. - m.Call = func(key string, _ any) (json.RawMessage, error) { - if key == steprun.AssignerHolders { - return json.RawMessage(`{"seats":[{"seat":"oci-artifact-registry","holders":[{"module":"mesh-registry","node":"novox"}]}],"holds":1}`), nil - } - return nil, errors.New("nats: no responders available for request") - } - if _, err := m.build(context.Background(), buildArgs()); err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "the holder is silent") { - t.Errorf("no responders with a holder named answered %v, want a transport fault", err) - } + if c.noSeat && (err == nil || !sdkstep.IsPassing(err) || !strings.Contains(err.Error(), "publish: ")) { + // A publish met by no responders is a transport fault with no seat call to learn why (design 47 H2 §799). + t.Errorf("no responders answered %v, want a transport fault opening publish:", err) } var archives []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 4f6a44a3f..d14d35aab 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -46,6 +46,13 @@ "description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", "minimum": 0.1, "default": 2 + }, + "again-delays": { + "type": "string", + "description": "The delays a step ask whose fetch or publish met its holder away is given again after: after its first run, then after its second, as durations such as 15s,60s (design 47 H2 §799); a third run met so ends the step failed with why fetch: or publish:.", + "trusted": false, + "runtime": true, + "default": "15s,60s" } }, "tools": [ @@ -79,7 +86,6 @@ "seat:oci-artifact-registry.publish-part", "seat:npm-package-registry.publish", "seat:npm-package-registry.publish-part", - "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module", "seat:mesh-catalog.seat-definitions" ],