BL17a: a key failed on a transport fault says it as 'transport fault: <class>, <seat> <node>, <cause>'; drops a committed binary
Builds: design 47 G §568, A-76 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -106,7 +105,7 @@ var PutTry = 10 * time.Second
|
||||
// §481, §600: `accept.<queue>.builder`). The put is a kept call (novox/hq ADR 0668
|
||||
// §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it
|
||||
// is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by
|
||||
// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it
|
||||
// PutTry, until ctx ends. A put that still meets one is an error opening `transport fault:` (TransportFault), so the key it
|
||||
// fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does.
|
||||
func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
|
||||
raw, err := buildstep.EncodeStepAsk(a)
|
||||
@@ -133,7 +132,8 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
|
||||
case err == nil:
|
||||
return nil
|
||||
case busretry.TimedOut(err) || busretry.Undelivered(err):
|
||||
return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err)
|
||||
seat, _ := buildstep.BuildSeat(a.Language)
|
||||
return errors.New(TransportFault(FaultClass(err.Error()), seat, "no node", fmt.Sprintf("the put of the %s ask %s: %v", a.Kind, a.ID, err)))
|
||||
}
|
||||
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
|
||||
}
|
||||
@@ -212,7 +212,8 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e
|
||||
if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) {
|
||||
// The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again
|
||||
// reads the key's last value and none of the runs before it.
|
||||
step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason)
|
||||
seat, _ := buildstep.BuildSeat(a.Language)
|
||||
step.Reason = TransportFault(FaultClass(step.Reason), seat, step.Node, step.Reason)
|
||||
}
|
||||
return step, nil
|
||||
}
|
||||
|
||||
@@ -72,11 +72,11 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
restore()
|
||||
switch {
|
||||
case c.transport:
|
||||
if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) {
|
||||
if err == nil || !strings.HasPrefix(err.Error(), TransportFaultPrefix+" deadline, node-build-go no node, ") {
|
||||
t.Errorf("%s: %v, want it opens with the transport fault's words", name, err)
|
||||
}
|
||||
case c.tries == 1:
|
||||
if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) {
|
||||
if err == nil || p.tries != 1 || strings.HasPrefix(err.Error(), TransportFaultPrefix) {
|
||||
t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries)
|
||||
}
|
||||
case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]:
|
||||
@@ -106,7 +106,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
}
|
||||
h.Write(st)
|
||||
}
|
||||
if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") {
|
||||
if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") {
|
||||
t.Errorf("three runs then failed: %v", err)
|
||||
}
|
||||
h = &FakeAsks{}
|
||||
@@ -139,7 +139,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
// how many runs; a drop that failed is said, not discarded.
|
||||
late := &FakeAsks{}
|
||||
late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"})
|
||||
if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") {
|
||||
if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") {
|
||||
t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err)
|
||||
}
|
||||
var said []string
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
)
|
||||
|
||||
// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: <class>, <seat> <node>,
|
||||
// <cause>`, the class one of ADR 0664 §1's four, the seat and node the step's, the cause the last run's in the build
|
||||
// module's words. No other failure opens `transport fault:`.
|
||||
|
||||
// TransportFaultPrefix opens the error of a key that failed on a transport fault.
|
||||
const TransportFaultPrefix = builds.TransportFaultPrefix
|
||||
|
||||
// The classes of a transport fault.
|
||||
const (
|
||||
ClassNoResponders = "no responders"
|
||||
ClassNameLookup = "name lookup"
|
||||
ClassConnection = "connection"
|
||||
ClassDeadline = "deadline"
|
||||
)
|
||||
|
||||
// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by
|
||||
// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut
|
||||
// connection, a connection.
|
||||
func FaultClass(cause string) string {
|
||||
c := strings.ToLower(cause)
|
||||
has := func(words ...string) bool {
|
||||
for _, w := range words {
|
||||
if strings.Contains(c, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"):
|
||||
return ClassNoResponders
|
||||
case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"):
|
||||
return ClassNameLookup
|
||||
case has("deadline", "timeout", "timed out", "etimedout"):
|
||||
return ClassDeadline
|
||||
}
|
||||
return ClassConnection
|
||||
}
|
||||
|
||||
// TransportFault is the error of a key that failed on a transport fault of the seat's step on node, with the cause of its
|
||||
// last run.
|
||||
func TransportFault(class, seat, node, cause string) string {
|
||||
return TransportFaultPrefix + " " + class + ", " + seat + " " + node + ", " + cause
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package buildrun
|
||||
|
||||
import "testing"
|
||||
|
||||
// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form.
|
||||
func TestFaultClassAndWords(t *testing.T) {
|
||||
for cause, class := range map[string]string{
|
||||
"nats: no responders available for request": ClassNoResponders,
|
||||
"the holder is silent: seat x answered": ClassNoResponders,
|
||||
"lookup proxy: no such host": ClassNameLookup,
|
||||
"Get https://r: context deadline exceeded": ClassDeadline,
|
||||
"read tcp: i/o timeout": ClassDeadline,
|
||||
"dial tcp: connection refused": ClassConnection,
|
||||
"ssh: kex_exchange_identification: read: reset": ClassConnection,
|
||||
} {
|
||||
if got := FaultClass(cause); got != class {
|
||||
t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class)
|
||||
}
|
||||
}
|
||||
if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want {
|
||||
t.Errorf("TransportFault = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -194,11 +194,15 @@ func FailedUntilChanged(err string) bool {
|
||||
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
|
||||
}
|
||||
|
||||
// FailedOnTransportFault says whether a failed key's error is a step's end after its third run met a transport fault
|
||||
// (buildstep.TransportFaultWords): a fault of the network, not of the source, so the builder does not retake it by itself
|
||||
// (ADR 0664 §3, which changes ADR 0429 clause 18 for this case); a person's hand act or a builder that starts again does.
|
||||
// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: <class>, <seat> <node>,
|
||||
// <cause>` (design 47 G §568, A-76). No other failure opens it.
|
||||
const TransportFaultPrefix = "transport fault:"
|
||||
|
||||
// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the
|
||||
// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder
|
||||
// that starts again, or an event that says its cause passed (design 47 G §568) builds it again.
|
||||
func FailedOnTransportFault(err string) bool {
|
||||
return strings.Contains(err, buildstep.TransportFaultWords)
|
||||
return strings.HasPrefix(err, TransportFaultPrefix)
|
||||
}
|
||||
|
||||
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
|
||||
|
||||
@@ -103,13 +103,13 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) {
|
||||
{VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false},
|
||||
{RefusedKind + "a package in go", false},
|
||||
{"go test: 2 failed", true},
|
||||
{"a transport fault on 3 runs (ace, g14, novox)", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's
|
||||
{"transport fault: deadline, node-build-go ace, i/o timeout", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's
|
||||
} {
|
||||
if got := FailedUntilChanged(c.err); got == c.again {
|
||||
t.Errorf("FailedUntilChanged(%q) = %v", c.err, got)
|
||||
}
|
||||
}
|
||||
if !FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace): i/o timeout") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") {
|
||||
if !FailedOnTransportFault("transport fault: connection, node-build-go ace, connection refused") || FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace)") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") {
|
||||
t.Error("FailedOnTransportFault must name the third run's end alone: the builder's own retake holds it (ADR 0664 §3)")
|
||||
}
|
||||
if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " {
|
||||
|
||||
@@ -166,7 +166,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) {
|
||||
take bool
|
||||
}{{ClaimOptions{}, false}, {ClaimOptions{ByHand: true}, true}, {ClaimOptions{TakeOver: true}, true}} {
|
||||
st = newMem()
|
||||
seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "mesh-build-go on ace: a transport fault on 3 runs (ace, g14, novox): lookup proxy: i/o timeout"})
|
||||
seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "transport fault: name lookup, node-build-go ace, lookup proxy: i/o timeout"})
|
||||
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", o.opt); err != nil || (got != nil) != o.take {
|
||||
t.Errorf("a transport fault on 3 runs with %+v: claim = %v, %v, want taken %v", o.opt, got, err, o.take)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Design 47 G §568: a key's error opens `transport fault:` only when every lane failed on one.
|
||||
func TestKeyErrorOpensTransportFaultOnlyWhenAllLanesDid(t *testing.T) {
|
||||
tf := "transport fault: deadline, node-build-go ace, i/o timeout"
|
||||
if got := keyError([]string{tf, "mesh-build-node on g14: the step failed: tsc: 2 errors"}); strings.HasPrefix(got, "transport fault:") {
|
||||
t.Errorf("beside a real failure the key opens %q", got)
|
||||
}
|
||||
if got := keyError([]string{tf, tf}); !strings.HasPrefix(got, "transport fault:") {
|
||||
t.Errorf("two transport faults open %q", got)
|
||||
}
|
||||
}
|
||||
@@ -855,6 +855,12 @@ var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.Refused
|
||||
// (`<tool> on <node>: the step <key> failed: not stored: ...`), the key's error opens with that prefix, so a reader of the
|
||||
// key need not search for it.
|
||||
func keyError(errs []string) string {
|
||||
// A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it,
|
||||
// so the key is not built again on an event for a failure the source made (design 47 G §568).
|
||||
errs = slices.Clone(errs)
|
||||
slices.SortStableFunc(errs, func(a, b string) int {
|
||||
return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix))
|
||||
})
|
||||
for _, prefix := range keyPrefixes {
|
||||
for i, e := range errs {
|
||||
if strings.HasPrefix(e, prefix) {
|
||||
@@ -991,11 +997,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil {
|
||||
// The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from
|
||||
// the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3).
|
||||
fault := ""
|
||||
l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err)
|
||||
if busretry.TimedOut(err) || busretry.Undelivered(err) {
|
||||
fault = sdkstep.TransportFaultWords + "the note of the ask: "
|
||||
seat, _ := sdkstep.BuildSeat(language)
|
||||
l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error()))
|
||||
}
|
||||
l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err)
|
||||
return
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
@@ -1017,6 +1023,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
}
|
||||
if step.State != sdkstep.StepPassed {
|
||||
l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason)
|
||||
if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) {
|
||||
l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568)
|
||||
}
|
||||
return
|
||||
}
|
||||
var answer sdkstep.BuildAnswer
|
||||
|
||||
Reference in New Issue
Block a user