B9: a fetch or a publish met away ends its run again, by again-delays, with no seat call

The build modules' shared steprun parts of §799: again-delays (15s,60s) read
into a Puller's Delay; a fetch's 401/5xx or connection fault and a publish's
no responders or exited bundle are transport faults whose why opens fetch: or
publish:; the assigner call that named a silent holder goes. The builder reads
an exited bundle as no responders and keeps the part in a key's fault words.

Builds: design 47 H2 §799 (H-A-390)
This commit is contained in:
2026-11-22 00:20:17 +01:00
parent 1058b0eec6
commit f6e7f4210b
12 changed files with 171 additions and 82 deletions
@@ -4,7 +4,6 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: <class>, <seat> <node>,
@@ -23,8 +22,9 @@ const (
)
// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by
// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut
// connection, a connection.
// (buildstep.PassingCause), read as no responders (a registry's bundle that exited included, design 47 H2 §799), a name
// lookup, a deadline or, for every other refused, reset or cut connection (a forge's 401 or 5xx included), a connection.
// A build module's why keeps its part, `fetch: ` or `publish: ` (§799), in the cause TransportFault says.
func FaultClass(cause string) string {
c := strings.ToLower(cause)
has := func(words ...string) bool {
@@ -36,7 +36,7 @@ func FaultClass(cause string) string {
return false
}
switch {
case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"):
case has("no responders", "bundle exited", "jetstream not enabled"):
return ClassNoResponders
case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"):
return ClassNameLookup
@@ -1,17 +1,23 @@
package buildrun
import "testing"
import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form.
func TestFaultClassAndWords(t *testing.T) {
for cause, class := range map[string]string{
"nats: no responders available for request": ClassNoResponders,
"the holder is silent: seat x answered": ClassNoResponders,
"lookup proxy: no such host": ClassNameLookup,
"Get https://r: context deadline exceeded": ClassDeadline,
"read tcp: i/o timeout": ClassDeadline,
"dial tcp: connection refused": ClassConnection,
"ssh: kex_exchange_identification: read: reset": ClassConnection,
"nats: no responders available for request": ClassNoResponders,
"publish: the bundle exited": ClassNoResponders,
"fetch: git fetch: The requested URL returned error: 401": ClassConnection,
"lookup proxy: no such host": ClassNameLookup,
"Get https://r: context deadline exceeded": ClassDeadline,
"read tcp: i/o timeout": ClassDeadline,
"dial tcp: connection refused": ClassConnection,
"ssh: kex_exchange_identification: read: reset": ClassConnection,
} {
if got := FaultClass(cause); got != class {
t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class)
@@ -20,4 +26,10 @@ func TestFaultClassAndWords(t *testing.T) {
if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want {
t.Errorf("TransportFault = %q, want %q", got, want)
}
// §799: a fetch's and a publish's why are said apart, each in the cause of its key's transport fault.
for _, why := range []string{"fetch: git fetch: HTTP 503", "publish: nats: no responders available for request"} {
if got := TransportFault(FaultClass(why), "node-build-go", "ace", why); !builds.FailedOnTransportFault(got) || !strings.HasSuffix(got, ", "+why) {
t.Errorf("TransportFault of %q = %q", why, got)
}
}
}
@@ -85,15 +85,6 @@ const (
AskMaxDeliver = AskRuns + 2
)
// AgainAfter is how long the bus waits before it gives an ask again after its run-th run ended on a transport fault: 1
// minute after the first, 4 after the second (ADR 0664 §2). It outlasts a per-source penalty of a forge's sshd.
func AgainAfter(run int) time.Duration {
if run < 1 {
run = 1
}
return time.Minute << (2 * (run - 1))
}
// StepAskVersion is the layout version of an ask.
const StepAskVersion = 1
@@ -500,7 +500,6 @@ func TestPauseCapacityAndComposerKey(t *testing.T) {
"npm error code ECONNRESET": true,
"fatal: unable to access the forge: Could not resolve host: mesh-forge": true,
"Get \"https://example.com/x.tar.gz\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)": true,
(&HolderSilent{Seat: "go-package-registry", Holder: "gopkgs", Node: "ace"}).Error(): true,
`invalid version: Get "https://proxy.golang.org/x/@v/y.info": dial tcp [2a00::1]:443: connect: network is unreachable`: true,
"dial tcp 10.10.0.1:22: connect: No route to host": true,
"nats: no responders available for request": false,
@@ -512,8 +511,8 @@ func TestPauseCapacityAndComposerKey(t *testing.T) {
t.Errorf("PassingCause(%q) = %v, want %v", words, !want, want)
}
}
if !IsPassing(fmt.Errorf("fetch: %w", &HolderSilent{Seat: "s"})) || IsPassing(errors.New("i/o timeout")) {
t.Error("a HolderSilent is a Passing error and a bare error is none")
if !IsPassing(fmt.Errorf("fetch: %w", &Passing{Err: errors.New("s")})) || IsPassing(errors.New("i/o timeout")) {
t.Error("a wrapped Passing is a Passing error and a bare error is none")
}
if Ended(Step{State: StepRunning}) || Ended(Step{State: StepAgain}) || !Ended(Step{State: StepPassed}) || !Ended(Step{State: StepFailed}) {
t.Error("Ended: running and again are no end")
@@ -522,9 +521,6 @@ func TestPauseCapacityAndComposerKey(t *testing.T) {
if raw, err := EncodeStep(Step{ID: "mvl0frv18d1990", Tool: "go ask", Node: "ace", State: StepDropped, Reason: "withdrawn"}); err != nil || !Ended(Step{State: StepDropped}) {
t.Errorf("a dropped step: %v (%s)", err, raw)
}
if AgainAfter(1) != time.Minute || AgainAfter(2) != 4*time.Minute {
t.Errorf("AgainAfter = %v, %v", AgainAfter(1), AgainAfter(2))
}
}
// The part `build-step` is one layout for the three build seats: each seat's fixtures read strictly in the neutral
+2 -17
View File
@@ -2,7 +2,6 @@ package buildstep
import (
"errors"
"fmt"
"strings"
)
@@ -21,26 +20,12 @@ func IsPassing(err error) bool {
return errors.As(err, &p)
}
// HolderSilentWords opens the words of a HolderSilent, so PassingCause reads them from a step's reason too.
const HolderSilentWords = "the holder is silent"
// HolderSilent is a seat verb that answered "no responders" while the assigner names a holder of the seat: its tool
// runner is being replaced, a moment later it answers. It is a Passing fault (ADR 0664 §1); a seat nothing holds is not.
type HolderSilent struct{ Seat, Holder, Node string }
func (h *HolderSilent) Error() string {
return fmt.Sprintf("%s: seat %s, held by %s on %s, answered no responders", HolderSilentWords, h.Seat, h.Holder, h.Node)
}
// Unwrap makes a HolderSilent a Passing.
func (h *HolderSilent) Unwrap() error { return &Passing{Err: errors.New(h.Error())} }
// passingWords are the words of a network that did not answer, lower-cased: go's and git's (a lookup, a timeout, a refused or
// reset connection, ssh's key exchange cut by a forge's per-source penalty) and npm's error codes. "no responders" alone is not
// one: it is the same for a seat nothing holds (HolderSilent is the case a holder is named).
// one: a publish's is the build module's to say (steprun.PublishFault, design 47 H2 §799).
var passingWords = []string{"i/o timeout", "no such host", "temporary failure in name resolution", "connection refused",
"connection reset", "tls handshake timeout", "network is unreachable", "no route to host", "lookup ", "kex_exchange_identification", "connection closed by",
"could not resolve host", "client.timeout", "operation timed out", "connection timed out", "eai_again", "etimedout", "econnreset", "econnrefused", "enotfound", strings.ToLower(HolderSilentWords)}
"could not resolve host", "client.timeout", "operation timed out", "connection timed out", "eai_again", "etimedout", "econnreset", "econnrefused", "enotfound"}
// PassingCause says words (a tool's output or a step's reason) are those of a transport fault.
func PassingCause(words string) bool {
@@ -0,0 +1,41 @@
package steprun
import (
"fmt"
"strings"
"time"
)
// AgainDelaysSetting is the setting each build module declares for the delay it passes with a run's `again` (design 47
// H2 §799): how long the bus waits before it gives an ask again after its first, then its second run ended on a
// transport fault, as Go durations separated by commas. DefaultAgainDelays is its default, so three runs span a
// one-minute absence of a holder.
const (
AgainDelaysSetting = "again-delays"
DefaultAgainDelays = "15s,60s"
)
// AgainDelays reads the setting again-delays into a Puller's Delay (§799): the run-th delay of the list, its last for a
// run past it; an empty setting is DefaultAgainDelays. A delay that is no positive duration is an error naming it.
func AgainDelays(setting string) (func(run int) time.Duration, error) {
if strings.TrimSpace(setting) == "" {
setting = DefaultAgainDelays
}
var delays []time.Duration
for _, w := range strings.Split(setting, ",") {
d, err := time.ParseDuration(strings.TrimSpace(w))
if err != nil || d <= 0 {
return nil, fmt.Errorf("the setting %s holds %q, whose %q is not a positive duration", AgainDelaysSetting, setting, w)
}
delays = append(delays, d)
}
return func(run int) time.Duration {
if run < 1 {
run = 1
}
return delays[min(run, len(delays))-1]
}, nil
}
// defaultAgainDelay is DefaultAgainDelays read, a Puller's delay when it names none.
var defaultAgainDelay, _ = AgainDelays(DefaultAgainDelays)
@@ -0,0 +1,74 @@
package steprun
import (
"context"
"errors"
"strings"
"testing"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// Design 47 H2 §799: a fetch met by a forge's 401 or a connection fault, and a publish met by no responders or a bundle
// that exited, end their run `again` after again-delays' delay, with no seat call; a third run's fault ends the step failed
// with the why `fetch: <words>` or `publish: <words>`.
func TestAFetchOrAPublishMetAwayIsATransportFault(t *testing.T) {
delay, err := AgainDelays("")
if err != nil || delay(1) != 15*time.Second || delay(2) != time.Minute || delay(3) != time.Minute {
t.Fatalf("the default again-delays: %v", err)
}
if own, err := AgainDelays("2s, 5s"); err != nil || own(1) != 2*time.Second || own(2) != 5*time.Second {
t.Errorf("a setting's own again-delays: %v", err)
}
if _, err := AgainDelays("15s,soon"); err == nil || !strings.Contains(err.Error(), AgainDelaysSetting) {
t.Errorf("a delay that is no duration is refused naming the setting: %v", err)
}
for _, tc := range []struct {
name, why string
err error
}{
{"a fetch met by a 401", "fetch: ", gitFailure("fetch", "fatal: unable to access the forge: The requested URL returned error: 401")},
{"a fetch met by a 503", "fetch: ", gitFailure("fetch", "remote: HTTP 503 store unavailable")},
{"a fetch met by a reset connection", "fetch: ", gitFailure("fetch", "kex_exchange_identification: read: Connection reset by peer")},
{"a publish met by no responders", "publish: ", PublishFault(errors.New("nats: no responders available for request"))},
{"a publish met by a bundle that exited", "publish: ", PublishFault(errors.New("the bundle exited"))},
{"a fetch of a commit that is not there", "", gitFailure("fetch", "fatal: unknown revision or path not in the working tree")},
{"a publish refused", "", PublishFault(errors.New("refused: the digest differs"))},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.why == "" {
if sdkstep.IsPassing(tc.err) {
t.Fatalf("%v is a transport fault, want a failure", tc.err)
}
return
}
if !sdkstep.IsPassing(tc.err) || !strings.HasPrefix(tc.err.Error(), tc.why) {
t.Fatalf("%v: want a transport fault opening with %q", tc.err, tc.why)
}
for _, prior := range []int{0, 2} { // its first run ends again after 15 s; its third ends the step failed
src, st := NewFakeAsks(), newSteps()
a := testAsk(sdkstep.AskDelivery)
_ = src.PublishAsk(a)
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
PullWait: 10 * time.Millisecond, Prior: func(string) int { return prior },
Run: func(context.Context, sdkstep.StepAsk) (any, error) { return nil, tc.err }}
ctx, cancel := context.WithCancel(context.Background())
go p.Pull(ctx)
key, _ := sdkstep.AskStepKey("", "shanks", a.ID)
waitFor(t, "the run's end", func() bool { s, ok := st.get(key); return ok && s.State != sdkstep.StepRunning })
cancel()
s, _ := st.get(key)
src.mu.Lock()
delays := append([]time.Duration(nil), src.Delays...)
src.mu.Unlock()
if prior == 0 && (s.State != sdkstep.StepAgain || len(delays) != 1 || delays[0] != 15*time.Second) {
t.Errorf("the first run: step %+v, delays %v, want again after 15s", s, delays)
}
if prior == 2 && (s.State != sdkstep.StepFailed || !strings.HasPrefix(s.Reason, tc.why)) {
t.Errorf("the third run: step %+v, want failed with the why %q", s, tc.why)
}
}
})
}
}
+9 -3
View File
@@ -54,15 +54,21 @@ var (
)
// gitFailure is a failed git command said in git's own words; one whose words are a network's (a lookup, a timeout, ssh's key
// exchange cut by a forge's per-source penalty) is a sdkstep.Passing, so the ask is given again after a delay (ADR 0664 §1).
// exchange cut by a forge's per-source penalty) or a forge's answer 401 or 5xx is a sdkstep.Passing whose words open with
// "fetch: ", so the ask is given again after a delay (ADR 0664 §1) and a third run's fault ends the step failed with the
// why `fetch: <words>` (design 47 H2 §799).
func gitFailure(verb, said string) error {
err := fmt.Errorf("git %s: %s", verb, said)
if sdkstep.PassingCause(said) {
return &sdkstep.Passing{Err: err}
if sdkstep.PassingCause(said) || forgeAnswered.MatchString(said) {
return &sdkstep.Passing{Err: fmt.Errorf("fetch: %w", err)}
}
return err
}
// forgeAnswered are git's words of a forge that answered 401 or a 5xx (§799): "The requested URL returned error: 502",
// "HTTP 503", "status 401".
var forgeAnswered = regexp.MustCompile(`(?i)(returned error|http|status)[: ]+(401|5[0-9][0-9])\b`)
// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached
// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh://<ssh name>/<owner>/<repo>.git, the ssh name being the
// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names
+10 -27
View File
@@ -15,7 +15,6 @@ import (
"os"
"strings"
assigner "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
imageclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient"
goclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/gopackageregistryclient"
npmclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/npmpackageregistryclient"
@@ -79,37 +78,21 @@ func SentHow(n int) func(parts.Sent) string {
// (the four registry seats share the publish verb's input, design 51), the parts are written by the client.
func PublishParts(ctx context.Context, call SeatCall, seat string, in reg.PublishInput, ps []parts.Part) (parts.Sent, error) {
sent, err := publishParts(ctx, call, seat, in, ps)
return sent, HolderSilentOr(call, seat, err)
return sent, PublishFault(err)
}
// AssignerHolders is the verb that tells who holds a seat: `mesh-module-assigner.assignments` with seats
// (the state seat-holders). The build modules' manifests `invokes` it.
const AssignerHolders = "seat:" + assigner.Seat + "." + assigner.VerbAssignments
// HolderSilentOr is err, or when it is a seat verb's "no responders" and the assigner names a holder of seat, a
// sdkstep.HolderSilent (a Passing: the holder's tool runner is being replaced and answers a moment later, ADR 0664 §1). The
// bus says "no responders" for a seat nothing holds too, so the words alone are no transport fault: with no holder named, or
// when the holders cannot be asked, the error stays as it is, said with why, and fails at once.
func HolderSilentOr(call SeatCall, seat string, err error) error {
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "no responders") {
// PublishFault is err, or, when a publish met no responders or a registry's bundle that exited, a sdkstep.Passing whose
// words open with "publish: " (design 47 H2 §799, ADR 0664 §1): the run ends `again`, with no seat call to learn why,
// and a third run's fault ends the step failed with the why `publish: <words>`. Any other error is returned as it is.
func PublishFault(err error) error {
if err == nil || sdkstep.IsPassing(err) {
return err
}
raw, aerr := call(AssignerHolders, map[string]any{"seats": assigner.AssignmentsInputSeatsTrue})
if aerr != nil {
return fmt.Errorf("%w (who holds %s could not be asked: %v)", err, seat, aerr)
words := strings.ToLower(err.Error())
if !strings.Contains(words, "no responders") && !strings.Contains(words, "bundle exited") {
return err
}
var answer struct {
Seats []assigner.SeatHolders `json:"seats"`
}
if jerr := json.Unmarshal(raw, &answer); jerr != nil {
return fmt.Errorf("%w (who holds %s was answered in what is not read: %v)", err, seat, jerr)
}
for _, sh := range answer.Seats {
if sh.Seat == seat && len(sh.Holders) > 0 {
return fmt.Errorf("%w: %w", err, &sdkstep.HolderSilent{Seat: seat, Holder: sh.Holders[0].Module, Node: sh.Holders[0].Node})
}
}
return fmt.Errorf("%w (nothing holds %s)", err, seat)
return &sdkstep.Passing{Err: fmt.Errorf("publish: %w", err)}
}
func publishParts(ctx context.Context, call SeatCall, seat string, in reg.PublishInput, ps []parts.Part) (parts.Sent, error) {
+3 -3
View File
@@ -75,8 +75,8 @@ type Puller struct {
Run func(ctx context.Context, ask sdkstep.StepAsk) (any, error)
// PullWait and Heartbeat default to DefaultPullWait and DefaultHeartbeat.
PullWait time.Duration
// Delay is how long the bus waits before it gives an ask again after its run-th run ended on a transport fault; nil is
// sdkstep.AgainAfter (a minute, then four). A test's own is short.
// Delay is how long the bus waits before it gives an ask again after its run-th run ended on a transport fault: the
// module's setting again-delays read by AgainDelays (design 47 H2 §799); nil is DefaultAgainDelays.
Delay func(run int) time.Duration
Heartbeat time.Duration
// Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once
@@ -820,7 +820,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key
if run := step.Run; err != nil && sdkstep.IsPassing(err) && run < sdkstep.AskRuns && msg.NakAfter != nil {
// A transport fault (ADR 0664 §1): the step says it is given again and the bus gives it after a delay, to this node or
// another; it is not acknowledged, so its asker sees no end. Run AskRuns times it ends failed, said as a transport fault.
delay := sdkstep.AgainAfter(run)
delay := defaultAgainDelay(run)
if p.Delay != nil {
delay = p.Delay(run)
}
@@ -207,7 +207,8 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) {
}
// ADR 0664 §1 with the bus's count above the runs that started: the ask was given back twice before its first run, so the
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after 4 minutes), not 3.
// bus says delivered 3 while the step key says 1 run began; a transport fault is run 2 (given again after again-delays'
// second, 60 s, design 47 H2 §799), not 3.
func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
src, st := NewFakeAsks(), newSteps()
src.Delivered = 3
@@ -227,8 +228,8 @@ func TestATransportFaultCountsRunsThatStartedNotDeliveries(t *testing.T) {
src.mu.Lock()
delays := append([]time.Duration(nil), src.Delays...)
src.mu.Unlock()
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != sdkstep.AgainAfter(2) {
t.Errorf("step %+v, delays %v, want again in run 2 after %s", s, delays, sdkstep.AgainAfter(2))
if s.State != sdkstep.StepAgain || s.Run != 2 || len(delays) != 1 || delays[0] != time.Minute {
t.Errorf("step %+v, delays %v, want again in run 2 after 60s", s, delays)
}
}
@@ -559,8 +559,8 @@ func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
passing bool
ackedOnly bool
}{
{"a transport fault in run 1", 1, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}, sdkstep.StepAgain, time.Minute, true, false},
{"a transport fault in run 2", 2, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}, sdkstep.StepAgain, 4 * time.Minute, true, false},
{"a transport fault in run 1", 1, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}, sdkstep.StepAgain, 15 * time.Second, true, false},
{"a transport fault in run 2", 2, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}, sdkstep.StepAgain, time.Minute, true, false},
{"a transport fault in the last run", sdkstep.AskRuns, &sdkstep.Passing{Err: errors.New("lookup proxy: i/o timeout")}, sdkstep.StepFailed, 0, true, true},
{"a compile error in run 1", 1, errors.New("undefined: x"), sdkstep.StepFailed, 0, false, true},
} {