Raise no condition for a take-in, a layout or a put cut by a cancellation (issue 1945)
Thirty urgent fingerprint-not-composed conditions were live on 2026-10-25, each a done key's take-in a stop or a restart cut with "context canceled". A cancellation is no fault of the build, the chain or the bucket: a take-in or a layout whose controller stops returns and the next one does it; one whose own step was cancelled is tried again without a condition; a send cut by a cancellation counts as no failure to put its files. The put test now starts and leaves the node's failure count at nothing, so a second run of it (-count=2) finds no count left by the first.
This commit is contained in:
@@ -80,7 +80,11 @@ func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
|||||||
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
if ctx.Err() != nil {
|
||||||
|
return // stopped: the next controller lays them out
|
||||||
|
}
|
||||||
|
// A cancelled layout is tried again and raises nothing: a cancellation is no fault of the chain (issue 1945).
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) {
|
||||||
o.Summary += err.Error()
|
o.Summary += err.Error()
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
failed = true
|
failed = true
|
||||||
|
|||||||
@@ -197,7 +197,9 @@ func followDoneKey(ctx context.Context, open *stores, module string) {
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return // stopped: a take-in cut by the stop is no fault of the build, and the next controller takes it in
|
return // stopped: a take-in cut by the stop is no fault of the build, and the next controller takes it in
|
||||||
}
|
}
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) {
|
||||||
|
// A take-in whose own step was cancelled is tried again, and raises nothing: a cancellation is no fault
|
||||||
|
// of the build (issue 1945: 30 urgent conditions live, each a take-in cut by a restart).
|
||||||
o.Summary += err.Error()
|
o.Summary += err.Error()
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
failed = true
|
failed = true
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A take-in cut by a cancellation raises no condition (issue 1945: on 2026-10-25 thirty urgent fingerprint-not-composed
|
||||||
|
// conditions were live, each a take-in a stop or a restart cut with "context canceled"): when the controller stops, the
|
||||||
|
// next one takes it in; when a step of its own was cancelled, it is tried again.
|
||||||
|
func TestATakeInCutByACancellationRaisesNothing(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
stop bool // the controller's context ends, else only the step's error is a cancellation
|
||||||
|
}{{"the controller stops", true}, {"a step was cancelled", false}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
was, wasRetry, wasSay, wasKeeper := doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom
|
||||||
|
t.Cleanup(func() {
|
||||||
|
doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom = was, wasRetry, wasSay, wasKeeper
|
||||||
|
})
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}})
|
||||||
|
t.Cleanup(func() { k.Close(context.Background()) })
|
||||||
|
conditionsFrom = k
|
||||||
|
doneKeyRetry = time.Millisecond
|
||||||
|
doneKeysSay = func(string) {}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
||||||
|
calls++
|
||||||
|
if tc.stop {
|
||||||
|
cancel() // stopped mid-way
|
||||||
|
return fmt.Errorf("the module cannot be composed: %w", context.Canceled)
|
||||||
|
}
|
||||||
|
if calls == 1 {
|
||||||
|
return fmt.Errorf("the module cannot be composed: %w", context.Canceled)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { takeDoneKey(ctx, nil, "cut-module"); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the take-in did not end")
|
||||||
|
}
|
||||||
|
if !tc.stop && calls != 2 {
|
||||||
|
t.Errorf("a take-in whose step was cancelled was tried %d time(s), want 2", calls)
|
||||||
|
}
|
||||||
|
if c, found, err := conditions.ReadOne(t.Context(), store, doneKeyObservation("cut-module").Key()); err != nil || found {
|
||||||
|
t.Fatalf("a take-in cut by a cancellation raised %+v (found %v, %v)", c, found, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -225,6 +225,9 @@ func filesNotPutObservation(node, summary string) conditions.Observation {
|
|||||||
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
||||||
// clears the count and the condition.
|
// clears the count and the condition.
|
||||||
func notePut(ctx context.Context, node string, put error) {
|
func notePut(ctx context.Context, node string, put error) {
|
||||||
|
if errors.Is(put, context.Canceled) || ctx.Err() != nil {
|
||||||
|
return // a send cut by a cancellation or a stop says nothing of the bucket (issue 1945)
|
||||||
|
}
|
||||||
filesFailMu.Lock()
|
filesFailMu.Lock()
|
||||||
if put == nil {
|
if put == nil {
|
||||||
delete(filesFailed, node)
|
delete(filesFailed, node)
|
||||||
|
|||||||
@@ -415,6 +415,7 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
|
|||||||
was := conditionsFrom
|
was := conditionsFrom
|
||||||
conditionsFrom = k
|
conditionsFrom = k
|
||||||
t.Cleanup(func() { conditionsFrom = was })
|
t.Cleanup(func() { conditionsFrom = was })
|
||||||
|
forgetPutFailures(t, "cond-node")
|
||||||
key := filesNotPutObservation("cond-node", "").Key()
|
key := filesNotPutObservation("cond-node", "").Key()
|
||||||
open := func() bool {
|
open := func() bool {
|
||||||
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||||
@@ -442,6 +443,36 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A send cut by a cancellation counts as no failure to put its files, however often (issue 1945: a cancellation is no fault
|
||||||
|
// of the bucket).
|
||||||
|
func TestAPutCutByACancellationRaisesNothing(t *testing.T) {
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}})
|
||||||
|
t.Cleanup(func() { k.Close(context.Background()) })
|
||||||
|
was := conditionsFrom
|
||||||
|
conditionsFrom = k
|
||||||
|
t.Cleanup(func() { conditionsFrom = was })
|
||||||
|
forgetPutFailures(t, "cut-node")
|
||||||
|
cut := fmt.Errorf("the files could not be put: %w", context.Canceled)
|
||||||
|
for range filesNotPutAfter + 1 {
|
||||||
|
notePut(t.Context(), "cut-node", cut)
|
||||||
|
}
|
||||||
|
if _, found, err := conditions.ReadOne(t.Context(), store, filesNotPutObservation("cut-node", "").Key()); err != nil || found {
|
||||||
|
t.Fatalf("sends cut by a cancellation raised the condition (found %v, %v)", found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetPutFailures starts the node's count of failed puts at nothing and leaves it so, as a run of the test again finds it.
|
||||||
|
func forgetPutFailures(t *testing.T, node string) {
|
||||||
|
forget := func() {
|
||||||
|
filesFailMu.Lock()
|
||||||
|
delete(filesFailed, node)
|
||||||
|
filesFailMu.Unlock()
|
||||||
|
}
|
||||||
|
forget()
|
||||||
|
t.Cleanup(forget)
|
||||||
|
}
|
||||||
|
|
||||||
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
||||||
type aBusThatSays struct {
|
type aBusThatSays struct {
|
||||||
link.Bus
|
link.Bus
|
||||||
|
|||||||
Reference in New Issue
Block a user