From c7e31640cf8f5154b09ee2163e535d7f05fa92a2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 16:29:56 +0100 Subject: [PATCH] Raise no condition for a take-in, a layout or a put cut by a cancellation (issue 1945) Thirty urgent fingerprint-not-composed conditions were live on 2026-10-25, each a done key's take-in a stop or a restart cut with "context canceled". A cancellation is no fault of the build, the chain or the bucket: a take-in or a layout whose controller stops returns and the next one does it; one whose own step was cancelled is tried again without a condition; a send cut by a cancellation counts as no failure to put its files. The put test now starts and leaves the node's failure count at nothing, so a second run of it (-count=2) finds no count left by the first. --- cmd/mesh-controller/chain.go | 6 +- cmd/mesh-controller/composed_keys.go | 4 +- .../composed_keys_cancel_test.go | 61 +++++++++++++++++++ cmd/mesh-controller/declaration_files.go | 3 + cmd/mesh-controller/declaration_files_test.go | 31 ++++++++++ 5 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 cmd/mesh-controller/composed_keys_cancel_test.go diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index 582fcff1..b03c07f4 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -80,7 +80,11 @@ func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) { Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "} if err != nil { fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err) - if keeper := conditionsFrom; keeper != nil && !failed { + if ctx.Err() != nil { + return // stopped: the next controller lays them out + } + // A cancelled layout is tried again and raises nothing: a cancellation is no fault of the chain (issue 1945). + if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) { o.Summary += err.Error() if _, oerr := keeper.Observe(ctx, o); oerr == nil { failed = true diff --git a/cmd/mesh-controller/composed_keys.go b/cmd/mesh-controller/composed_keys.go index 36a4a2cb..c2d31118 100644 --- a/cmd/mesh-controller/composed_keys.go +++ b/cmd/mesh-controller/composed_keys.go @@ -197,7 +197,9 @@ func followDoneKey(ctx context.Context, open *stores, module string) { if ctx.Err() != nil { return // stopped: a take-in cut by the stop is no fault of the build, and the next controller takes it in } - if keeper := conditionsFrom; keeper != nil && !failed { + if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) { + // A take-in whose own step was cancelled is tried again, and raises nothing: a cancellation is no fault + // of the build (issue 1945: 30 urgent conditions live, each a take-in cut by a restart). o.Summary += err.Error() if _, oerr := keeper.Observe(ctx, o); oerr == nil { failed = true diff --git a/cmd/mesh-controller/composed_keys_cancel_test.go b/cmd/mesh-controller/composed_keys_cancel_test.go new file mode 100644 index 00000000..34292364 --- /dev/null +++ b/cmd/mesh-controller/composed_keys_cancel_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// A take-in cut by a cancellation raises no condition (issue 1945: on 2026-10-25 thirty urgent fingerprint-not-composed +// conditions were live, each a take-in a stop or a restart cut with "context canceled"): when the controller stops, the +// next one takes it in; when a step of its own was cancelled, it is tried again. +func TestATakeInCutByACancellationRaisesNothing(t *testing.T) { + for _, tc := range []struct { + name string + stop bool // the controller's context ends, else only the step's error is a cancellation + }{{"the controller stops", true}, {"a step was cancelled", false}} { + t.Run(tc.name, func(t *testing.T) { + was, wasRetry, wasSay, wasKeeper := doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom + t.Cleanup(func() { + doneKeyTakeOnce, doneKeyRetry, doneKeysSay, conditionsFrom = was, wasRetry, wasSay, wasKeeper + }) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}}) + t.Cleanup(func() { k.Close(context.Background()) }) + conditionsFrom = k + doneKeyRetry = time.Millisecond + doneKeysSay = func(string) {} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + doneKeyTakeOnce = func(context.Context, *stores, string) error { + calls++ + if tc.stop { + cancel() // stopped mid-way + return fmt.Errorf("the module cannot be composed: %w", context.Canceled) + } + if calls == 1 { + return fmt.Errorf("the module cannot be composed: %w", context.Canceled) + } + return nil + } + done := make(chan struct{}) + go func() { takeDoneKey(ctx, nil, "cut-module"); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the take-in did not end") + } + if !tc.stop && calls != 2 { + t.Errorf("a take-in whose step was cancelled was tried %d time(s), want 2", calls) + } + if c, found, err := conditions.ReadOne(t.Context(), store, doneKeyObservation("cut-module").Key()); err != nil || found { + t.Fatalf("a take-in cut by a cancellation raised %+v (found %v, %v)", c, found, err) + } + }) + } +} diff --git a/cmd/mesh-controller/declaration_files.go b/cmd/mesh-controller/declaration_files.go index c87902ab..888e4eed 100644 --- a/cmd/mesh-controller/declaration_files.go +++ b/cmd/mesh-controller/declaration_files.go @@ -225,6 +225,9 @@ func filesNotPutObservation(node, summary string) conditions.Observation { // notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success // clears the count and the condition. func notePut(ctx context.Context, node string, put error) { + if errors.Is(put, context.Canceled) || ctx.Err() != nil { + return // a send cut by a cancellation or a stop says nothing of the bucket (issue 1945) + } filesFailMu.Lock() if put == nil { delete(filesFailed, node) diff --git a/cmd/mesh-controller/declaration_files_test.go b/cmd/mesh-controller/declaration_files_test.go index e96083ce..901d1a71 100644 --- a/cmd/mesh-controller/declaration_files_test.go +++ b/cmd/mesh-controller/declaration_files_test.go @@ -415,6 +415,7 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi was := conditionsFrom conditionsFrom = k t.Cleanup(func() { conditionsFrom = was }) + forgetPutFailures(t, "cond-node") key := filesNotPutObservation("cond-node", "").Key() open := func() bool { _, found, err := conditions.ReadOne(t.Context(), store, key) @@ -442,6 +443,36 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi } } +// A send cut by a cancellation counts as no failure to put its files, however often (issue 1945: a cancellation is no fault +// of the bucket). +func TestAPutCutByACancellationRaisesNothing(t *testing.T) { + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}}) + t.Cleanup(func() { k.Close(context.Background()) }) + was := conditionsFrom + conditionsFrom = k + t.Cleanup(func() { conditionsFrom = was }) + forgetPutFailures(t, "cut-node") + cut := fmt.Errorf("the files could not be put: %w", context.Canceled) + for range filesNotPutAfter + 1 { + notePut(t.Context(), "cut-node", cut) + } + if _, found, err := conditions.ReadOne(t.Context(), store, filesNotPutObservation("cut-node", "").Key()); err != nil || found { + t.Fatalf("sends cut by a cancellation raised the condition (found %v, %v)", found, err) + } +} + +// forgetPutFailures starts the node's count of failed puts at nothing and leaves it so, as a run of the test again finds it. +func forgetPutFailures(t *testing.T, node string) { + forget := func() { + filesFailMu.Lock() + delete(filesFailed, node) + filesFailMu.Unlock() + } + forget() + t.Cleanup(forget) +} + // aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held. type aBusThatSays struct { link.Bus