Issue 241 withdrew seven consumers in one pass on one misread file; its fix refuses a file it cannot read, and a file read whole that names nobody still withdraws everybody. A pass that would withdraw more than one consumer at once, or more than half of those it holds, now withdraws nothing: each consumer it kept is announced provisioner.failing with the class withdrawal-braked, so the controller raises it as a condition and the operator is told, and one is let go each hour while the mesh goes on not asking for them. A consumer asked for again is kept and said recovered. Postgres and keycloak carry the harness identically.
114 lines
3.6 KiB
Go
114 lines
3.6 KiB
Go
package main
|
|
|
|
// The withdrawal brake (novox/hq to-be 45 Phase 2, ADR 0227 rule 4; replay R6 of issue 241): a pass that
|
|
// would withdraw more than its bound withdraws nothing, says so, and announces every consumer it kept as
|
|
// failing with the class withdrawal-braked, which the controller raises as a condition; one is let go
|
|
// every hour while the mesh goes on not asking; a consumer asked for again is kept and said recovered.
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// sevenConsumers is the control node's postgres on 2026-10-04: seven databases, all in use.
|
|
func sevenConsumers(w *world) {
|
|
var given []map[string]any
|
|
for i := 1; i <= 7; i++ {
|
|
given = append(given, map[string]any{"as": fmt.Sprintf("consumer%d", i), "node": "anchor"})
|
|
}
|
|
w.give(given...)
|
|
w.h.Reconcile(ctx)
|
|
}
|
|
|
|
func TestAWithdrawalOfEveryConsumerIsBrakedAndSaid(t *testing.T) {
|
|
w, said := standingWorld(t)
|
|
sevenConsumers(w)
|
|
// A file read whole that names nobody: the shape of 241 that its first fix does not catch.
|
|
w.give()
|
|
w.passes(6 * time.Minute)
|
|
if len(w.a.removed) != 0 {
|
|
t.Fatalf("a pass over its bound withdrew %v", w.a.removed)
|
|
}
|
|
braked := 0
|
|
for _, a := range *said {
|
|
if a.event == EventFailing && a.body["class"] == ClassWithdrawalBraked && a.body["node"] == "anchor" {
|
|
braked++
|
|
}
|
|
}
|
|
if braked != 7 {
|
|
t.Fatalf("%d of the 7 consumers kept were announced as braked: %v", braked, *said)
|
|
}
|
|
if !strings.Contains(strings.Join(w.said, "\n"), "WITHDRAWAL BRAKED") {
|
|
t.Fatal("the brake was not said")
|
|
}
|
|
|
|
// One is let go once the brake has held an hour, and then one an hour.
|
|
w.passes(55 * time.Minute)
|
|
if len(w.a.removed) != 1 {
|
|
t.Fatalf("after an hour of the mesh not asking, %d were withdrawn, want 1: %v", len(w.a.removed), w.a.removed)
|
|
}
|
|
w.passes(59 * time.Minute)
|
|
if len(w.a.removed) != 1 {
|
|
t.Fatalf("a second was let go within the hour: %v", w.a.removed)
|
|
}
|
|
w.passes(2 * time.Minute)
|
|
if len(w.a.removed) != 2 {
|
|
t.Fatalf("the second was not let go after another hour: %v", w.a.removed)
|
|
}
|
|
}
|
|
|
|
func TestAConsumerAskedForAgainIsKeptAndNotWithdrawn(t *testing.T) {
|
|
w, said := standingWorld(t)
|
|
sevenConsumers(w)
|
|
w.give()
|
|
w.passes(6 * time.Minute)
|
|
// The file is right again: every consumer is asked for, nothing was withdrawn, each is recovered.
|
|
sevenConsumers(w)
|
|
w.passes(5 * time.Second)
|
|
if len(w.a.removed) != 0 {
|
|
t.Fatalf("withdrew %v", w.a.removed)
|
|
}
|
|
recovered := 0
|
|
for _, a := range *said {
|
|
if a.event == EventRecovered && a.body["why"] == nil {
|
|
recovered++
|
|
}
|
|
}
|
|
if recovered != 7 {
|
|
t.Fatalf("%d of the 7 kept consumers were said recovered: %v", recovered, *said)
|
|
}
|
|
if len(w.h.braked) != 0 {
|
|
t.Fatalf("the brake still holds %v", w.h.braked)
|
|
}
|
|
}
|
|
|
|
// Within the bound — one consumer of several, or the last one held — a withdrawal goes as asked.
|
|
func TestAWithdrawalWithinItsBoundIsNotBraked(t *testing.T) {
|
|
w := newWorld(t)
|
|
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}, map[string]any{"as": "c"})
|
|
w.h.Reconcile(ctx)
|
|
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
|
w.h.Reconcile(ctx)
|
|
if strings.Join(w.a.removed, ",") != "c" {
|
|
t.Fatalf("one of three was not withdrawn: %v", w.a.removed)
|
|
}
|
|
// Two of the remaining two at once is over the bound.
|
|
w.give()
|
|
w.h.Reconcile(ctx)
|
|
if strings.Join(w.a.removed, ",") != "c" {
|
|
t.Fatalf("two at once were withdrawn: %v", w.a.removed)
|
|
}
|
|
for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}} {
|
|
if w.h.overTheBound(c.n, c.held) {
|
|
t.Errorf("%d of %d is over the bound", c.n, c.held)
|
|
}
|
|
}
|
|
for _, c := range []struct{ n, held int }{{2, 2}, {2, 7}, {7, 7}} {
|
|
if !w.h.overTheBound(c.n, c.held) {
|
|
t.Errorf("%d of %d is within the bound", c.n, c.held)
|
|
}
|
|
}
|
|
}
|