diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index bbc793fb..7db40ad7 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log" - "strings" "sync" "time" @@ -106,7 +105,7 @@ var PutTry = 10 * time.Second // §481, §600: `accept..builder`). The put is a kept call (novox/hq ADR 0668 // §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it // is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by -// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it +// PutTry, until ctx ends. A put that still meets one is an error opening `transport fault:` (TransportFault), so the key it // fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does. func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { raw, err := buildstep.EncodeStepAsk(a) @@ -133,7 +132,8 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { case err == nil: return nil case busretry.TimedOut(err) || busretry.Undelivered(err): - return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err) + seat, _ := buildstep.BuildSeat(a.Language) + return errors.New(TransportFault(FaultClass(err.Error()), seat, "no node", fmt.Sprintf("the put of the %s ask %s: %v", a.Kind, a.ID, err))) } return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) } @@ -212,7 +212,8 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) { // The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again // reads the key's last value and none of the runs before it. - step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason) + seat, _ := buildstep.BuildSeat(a.Language) + step.Reason = TransportFault(FaultClass(step.Reason), seat, step.Node, step.Reason) } return step, nil } diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index ed127a56..e31fd97e 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -72,11 +72,11 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { restore() switch { case c.transport: - if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) { + if err == nil || !strings.HasPrefix(err.Error(), TransportFaultPrefix+" deadline, node-build-go no node, ") { t.Errorf("%s: %v, want it opens with the transport fault's words", name, err) } case c.tries == 1: - if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) { + if err == nil || p.tries != 1 || strings.HasPrefix(err.Error(), TransportFaultPrefix) { t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries) } case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]: @@ -106,7 +106,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { } h.Write(st) } - if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") { + if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") { t.Errorf("three runs then failed: %v", err) } h = &FakeAsks{} @@ -139,7 +139,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // how many runs; a drop that failed is said, not discarded. late := &FakeAsks{} late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"}) - if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") { + if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") { t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err) } var said []string diff --git a/modules/mesh-delivery-builder/buildrun/faults.go b/modules/mesh-delivery-builder/buildrun/faults.go new file mode 100644 index 00000000..e1054451 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults.go @@ -0,0 +1,53 @@ +package buildrun + +import ( + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: , , +// `, the class one of ADR 0664 §1's four, the seat and node the step's, the cause the last run's in the build +// module's words. No other failure opens `transport fault:`. + +// TransportFaultPrefix opens the error of a key that failed on a transport fault. +const TransportFaultPrefix = builds.TransportFaultPrefix + +// The classes of a transport fault. +const ( + ClassNoResponders = "no responders" + ClassNameLookup = "name lookup" + ClassConnection = "connection" + ClassDeadline = "deadline" +) + +// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by +// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut +// connection, a connection. +func FaultClass(cause string) string { + c := strings.ToLower(cause) + has := func(words ...string) bool { + for _, w := range words { + if strings.Contains(c, w) { + return true + } + } + return false + } + switch { + case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"): + return ClassNoResponders + case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"): + return ClassNameLookup + case has("deadline", "timeout", "timed out", "etimedout"): + return ClassDeadline + } + return ClassConnection +} + +// TransportFault is the error of a key that failed on a transport fault of the seat's step on node, with the cause of its +// last run. +func TransportFault(class, seat, node, cause string) string { + return TransportFaultPrefix + " " + class + ", " + seat + " " + node + ", " + cause +} diff --git a/modules/mesh-delivery-builder/buildrun/faults_test.go b/modules/mesh-delivery-builder/buildrun/faults_test.go new file mode 100644 index 00000000..528b0699 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults_test.go @@ -0,0 +1,23 @@ +package buildrun + +import "testing" + +// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form. +func TestFaultClassAndWords(t *testing.T) { + for cause, class := range map[string]string{ + "nats: no responders available for request": ClassNoResponders, + "the holder is silent: seat x answered": ClassNoResponders, + "lookup proxy: no such host": ClassNameLookup, + "Get https://r: context deadline exceeded": ClassDeadline, + "read tcp: i/o timeout": ClassDeadline, + "dial tcp: connection refused": ClassConnection, + "ssh: kex_exchange_identification: read: reset": ClassConnection, + } { + if got := FaultClass(cause); got != class { + t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class) + } + } + if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want { + t.Errorf("TransportFault = %q, want %q", got, want) + } +} diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index fe8d7f7b..f06ef70a 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -194,11 +194,15 @@ func FailedUntilChanged(err string) bool { return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) } -// FailedOnTransportFault says whether a failed key's error is a step's end after its third run met a transport fault -// (buildstep.TransportFaultWords): a fault of the network, not of the source, so the builder does not retake it by itself -// (ADR 0664 §3, which changes ADR 0429 clause 18 for this case); a person's hand act or a builder that starts again does. +// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: , , +// ` (design 47 G §568, A-76). No other failure opens it. +const TransportFaultPrefix = "transport fault:" + +// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the +// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder +// that starts again, or an event that says its cause passed (design 47 G §568) builds it again. func FailedOnTransportFault(err string) bool { - return strings.Contains(err, buildstep.TransportFaultWords) + return strings.HasPrefix(err, TransportFaultPrefix) } // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and diff --git a/modules/mesh-delivery-builder/builds/reference_test.go b/modules/mesh-delivery-builder/builds/reference_test.go index 1713338f..89b6d58a 100644 --- a/modules/mesh-delivery-builder/builds/reference_test.go +++ b/modules/mesh-delivery-builder/builds/reference_test.go @@ -103,13 +103,13 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) { {VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false}, {RefusedKind + "a package in go", false}, {"go test: 2 failed", true}, - {"a transport fault on 3 runs (ace, g14, novox)", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's + {"transport fault: deadline, node-build-go ace, i/o timeout", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's } { if got := FailedUntilChanged(c.err); got == c.again { t.Errorf("FailedUntilChanged(%q) = %v", c.err, got) } } - if !FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace): i/o timeout") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { + if !FailedOnTransportFault("transport fault: connection, node-build-go ace, connection refused") || FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace)") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { t.Error("FailedOnTransportFault must name the third run's end alone: the builder's own retake holds it (ADR 0664 §3)") } if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " { diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index eeda081f..c553addd 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -166,7 +166,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { take bool }{{ClaimOptions{}, false}, {ClaimOptions{ByHand: true}, true}, {ClaimOptions{TakeOver: true}, true}} { st = newMem() - seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "mesh-build-go on ace: a transport fault on 3 runs (ace, g14, novox): lookup proxy: i/o timeout"}) + seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "transport fault: name lookup, node-build-go ace, lookup proxy: i/o timeout"}) if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", o.opt); err != nil || (got != nil) != o.take { t.Errorf("a transport fault on 3 runs with %+v: claim = %v, %v, want taken %v", o.opt, got, err, o.take) } diff --git a/modules/mesh-delivery-builder/internal/build/keyerror_test.go b/modules/mesh-delivery-builder/internal/build/keyerror_test.go new file mode 100644 index 00000000..583d93ac --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/keyerror_test.go @@ -0,0 +1,17 @@ +package build + +import ( + "strings" + "testing" +) + +// Design 47 G §568: a key's error opens `transport fault:` only when every lane failed on one. +func TestKeyErrorOpensTransportFaultOnlyWhenAllLanesDid(t *testing.T) { + tf := "transport fault: deadline, node-build-go ace, i/o timeout" + if got := keyError([]string{tf, "mesh-build-node on g14: the step failed: tsc: 2 errors"}); strings.HasPrefix(got, "transport fault:") { + t.Errorf("beside a real failure the key opens %q", got) + } + if got := keyError([]string{tf, tf}); !strings.HasPrefix(got, "transport fault:") { + t.Errorf("two transport faults open %q", got) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ff63d787..e03214a6 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -855,6 +855,12 @@ var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.Refused // (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the // key need not search for it. func keyError(errs []string) string { + // A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it, + // so the key is not built again on an event for a failure the source made (design 47 G §568). + errs = slices.Clone(errs) + slices.SortStableFunc(errs, func(a, b string) int { + return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix)) + }) for _, prefix := range keyPrefixes { for i, e := range errs { if strings.HasPrefix(e, prefix) { @@ -991,11 +997,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil { // The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from // the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3). - fault := "" + l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err) if busretry.TimedOut(err) || busretry.Undelivered(err) { - fault = sdkstep.TransportFaultWords + "the note of the ask: " + seat, _ := sdkstep.BuildSeat(language) + l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error())) } - l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err) return } if err := b.Asks.Put(ctx, ask); err != nil { @@ -1017,6 +1023,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } if step.State != sdkstep.StepPassed { l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason) + if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) { + l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568) + } return } var answer sdkstep.BuildAnswer